Skip to content

Commit

Permalink
Commented out parquet commands to check old sqlite outputs
Browse files Browse the repository at this point in the history
  • Loading branch information
alexglasertpx committed Nov 26, 2024
1 parent b0b391c commit b8c5b04
Showing 1 changed file with 17 additions and 17 deletions.
34 changes: 17 additions & 17 deletions digital_land/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -395,23 +395,23 @@ def dataset_create(

package.add_counts()

# Repeat for parquet
# Set up cache directory to store parquet files. The sqlite files created from this will be saved in the dataset
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)

pqpackage = DatasetParquetPackage(
dataset,
path=output_path,
input_paths=input_paths,
specification_dir=None, # TBD: package should use this specification object
)
pqpackage.create_temp_table(input_paths)
pqpackage.load_facts(input_paths, cache_dir)
pqpackage.load_fact_resource(input_paths, cache_dir)
pqpackage.load_entities(input_paths, cache_dir, organisation_path)
pqpackage.pq_to_sqlite(output_path, cache_dir)
pqpackage.close_conn()
# # Repeat for parquet
# # Set up cache directory to store parquet files. The sqlite files created from this will be saved in the dataset
# if not os.path.exists(cache_dir):
# os.makedirs(cache_dir)
#
# pqpackage = DatasetParquetPackage(
# dataset,
# path=output_path,
# input_paths=input_paths,
# specification_dir=None, # TBD: package should use this specification object
# )
# pqpackage.create_temp_table(input_paths)
# pqpackage.load_facts(input_paths, cache_dir)
# pqpackage.load_fact_resource(input_paths, cache_dir)
# pqpackage.load_entities(input_paths, cache_dir, organisation_path)
# pqpackage.pq_to_sqlite(output_path, cache_dir)
# pqpackage.close_conn()


def dataset_dump(input_path, output_path):
Expand Down

0 comments on commit b8c5b04

Please sign in to comment.