Draft: Rewrites whole data processing

Details:
- Only OpenAlex is now supported
- The Home component offers both JSON files upload and an iframe to
  directly navigate OpenAlex
- The App component has been drastically simplified
- Simplified a lot data processing
- Removes lots of no more used code
- Removes reports related code

TODO.md

@@ -1,19 +1,8 @@
-## Développement
+### TODO
 
-- [ ] Streamer les CSVs entrée et récupérer `dataset`
-- [ ] Valider le schéma des données
-- [ ] Créer le graphe total `graph` dans Graphology à partir de `dataset`
-- [ ] Extraire les volumétries
-- [ ] Filtrer le graphe total dans `filteredGraph`
-- [ ] Spatialiser le graphe filtré
-- [ ] Afficher le graphe filtré
-- [ ] Exporter le graphe filtré et spatialisé pour Gephi (CSV ?)
-
-## Écrans
-
-- [ ] Accueil - Drag'n'drop des fichier d'entrée
-- [ ] Affichage de l'état du parsing des fichiers
-- [ ] Affichage des volumétries
-- [ ] Définition des filtrages
-- [ ] Loader le temps des calculs graphes (FA2 en particulier)
-- [ ] Affichage du graphe + Bouton télécharger
+- [ ] Fix refs labels
+- [ ] Fix work nodes
+- [ ] Fix JSON import
+- [ ] Add missing fields
+- [ ] Sample refs nodes by positions
+- [ ] Handle caching in Home component

src/Components/App.tsx

@@ -1,130 +1,81 @@
-import Graph, { UndirectedGraph } from "graphology";
-import { isEmpty, pickBy, toPairs } from "lodash";
-import React, { FC, useEffect, useState } from "react";
+import Graph from "graphology";
+import React, { FC, useCallback, useState } from "react";
 import { FaSpinner } from "react-icons/fa6";
 
+import { indexWorks } from "../Lib/data";
+import { getDefaultFilters, getFilteredGraph } from "../Lib/filters";
 import { aggregateFieldIndices } from "../Lib/getAggregations";
-import { indexCSVs } from "../Lib/indexCSVs";
-import { loadFilterGraph } from "../Lib/loadFilterGraph";
 import { prepareGraph } from "../Lib/prepareGraph";
-import { CSVFormat, FieldIndices, FiltersType } from "../Lib/types";
+import { Aggregations, FieldIndices, FiltersType, Work } from "../Lib/types";
 import "./App.css";
 import Filters from "./Filters";
 import Home from "./Home";
 import Viz from "./Viz";
 
-//TODO: put this in config
-const defaultFilters: FiltersType = {
-  references: 2,
-};
-
 const App: FC = () => {
-  const [files, setFiles] = useState<File[] | null>(null);
-  const [filesReady, setFilesReady] = useState<boolean>(false);
-  const [range, setRange] = useState<{ min?: number; max?: number }>({});
-  const [format, setCSVFormat] = useState<CSVFormat | null>(null);
-  const [filters, setFilters] = useState<FiltersType>(defaultFilters);
-  const [filtersReady, setFiltersReady] = useState<boolean>(false);
-  const [indices, setIndices] = useState<FieldIndices>({});
+  const [data, setData] = useState<{
+    works: Work[];
+    indices: FieldIndices;
+    aggregations: Aggregations;
+    filters: FiltersType;
+  } | null>(null);
   const [filteredGraph, setFilteredGraph] = useState<Graph | null>(null);
   const [isLoading, setIsLoading] = useState<boolean>(false);
   const [loaderMessage, setLoaderMessage] = useState<string | null>(null);
 
-  // ****************** Index uploaded CSV prepare filters *************** //
-  useEffect(() => {
-    // Load CSVs on submit Home component:
-    if (filesReady && files && files.length && format && isEmpty(indices) && !isLoading) {
+  const prepareData = useCallback(
+    async (works: Work[]) => {
       setIsLoading(true);
-      indexCSVs(files, format, range, setLoaderMessage).then((indices) => {
-        setIndices(indices);
-        setIsLoading(false);
-      });
-    }
-  }, [filesReady, files, format, indices, isLoading, range]);
+      const indices = await indexWorks(works);
+      const aggregations = aggregateFieldIndices(indices);
 
-  // ****************** Build and display Network *********************** //
-  useEffect(() => {
-    // Prepare graph when filters are submitted:
-    if (files && files.length && format && indices && filters && filtersReady && !filteredGraph && !isLoading) {
-      setIsLoading(true);
-      // filter fieldIndices
-      // hash with at least one dups are passed for filtering
-      const filteredFieldIndices: FieldIndices = {
-        hash: pickBy(indices.hash, (nb) => nb > 1),
-      };
-      // keep the occ index entries only if occ counts >= filters
-      toPairs(indices).forEach(([fieldType, valuesOccs]) => {
-        if (filters[fieldType])
-          filteredFieldIndices[fieldType] = pickBy(valuesOccs, (occ) => occ >= filters[fieldType]);
+      setIsLoading(false);
+      setData({
+        works,
+        indices,
+        aggregations,
+        filters: getDefaultFilters(aggregations),
       });
+    },
+    [setIsLoading, setData],
+  );
 
-      setLoaderMessage("Creating the graph from CSVs...");
-      loadFilterGraph(files, format, filteredFieldIndices, range, setLoaderMessage).then((graph: UndirectedGraph) => {
-        setLoaderMessage("Spacialiazing the graph...");
-        prepareGraph(graph).then((spacialisedGraph) => {
-          setFilteredGraph(spacialisedGraph);
-          setIsLoading(false);
-        });
-      });
-    }
-  }, [files, filteredGraph, filters, format, indices, isLoading, range, filtersReady]);
+  const filterGraph = useCallback(async () => {
+    if (!data) return;
+
+    setIsLoading(true);
+    let graph = await getFilteredGraph(data.works, data.indices, data.filters);
+    graph = await prepareGraph(graph);
+
+    setIsLoading(false);
+    setFilteredGraph(graph);
+  }, [data, setIsLoading, setFilteredGraph]);
 
-  // ****************** Chose the right component *********************** //
   let Component = <div>Woops, something went wrong...</div>;
 
-  if (isEmpty(indices))
-    Component = (
-      <Home
-        files={files || []}
-        range={range}
-        format={format}
-        onSubmit={(files: File[], format: CSVFormat, range: { min?: number; max?: number }) => {
-          setFiles(files);
-          setRange(range);
-          setCSVFormat(format);
-          setFilesReady(true);
-        }}
-      />
-    );
-  if (format && !isEmpty(indices) && !filteredGraph) {
-    // Aggregate data:
-    const { aggregations, fields } = aggregateFieldIndices(indices, format);
-    const articlesMetadata = toPairs(indices.hash).reduce(
-      (r, [, nb]) => ({
-        nbArticles: r.nbArticles + nb,
-        nbDuplicates: r.nbDuplicates + nb - 1,
-      }),
-      { nbArticles: 0, nbDuplicates: 0 },
-    );
+  if (!data) Component = <Home onSubmit={prepareData} />;
+  if (data && !filteredGraph) {
     Component = (
       <Filters
-        filters={filters}
-        setFilters={setFilters}
-        aggregations={aggregations}
-        fields={fields}
-        onSubmit={() => {
-          setFiltersReady(true);
-        }}
-        articlesMetadata={articlesMetadata}
-        range={range}
+        works={data.works}
+        aggregations={data.aggregations}
+        filters={data.filters}
+        setFilters={(filters) => setData({ ...data, filters })}
+        onSubmit={filterGraph}
         onGoBack={() => {
-          setIndices({});
-          setFilters(defaultFilters);
-          setFilesReady(false);
+          setData(null);
         }}
       />
     );
   }
-  if (filteredGraph)
+  if (data && filteredGraph)
     Component = (
       <Viz
         graph={filteredGraph}
-        indices={indices}
-        filters={filters || {}}
-        format={format as CSVFormat}
+        indices={data.indices}
+        filters={data.filters}
         onGoBack={() => {
           setFilteredGraph(null);
-          setFiltersReady(false);
         }}
       />
     );