Skip to content

Commit

Permalink
Revert "Go back to non-parquet loading. (#283)" (#286)
Browse files Browse the repository at this point in the history
This reverts commit d27787f.
  • Loading branch information
cjohns-scottlogic authored Nov 26, 2024
1 parent d27787f commit beeaf1c
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 12 deletions.
15 changes: 6 additions & 9 deletions digital_land/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -379,10 +379,8 @@ def dataset_create(
package.create()
for path in input_paths:
path_obj = Path(path)
package.load_transformed(path) # Remove this to use Parquet
package.load_column_fields(column_field_dir / dataset / path_obj.name)
package.load_dataset_resource(dataset_resource_dir / dataset / path_obj.name)
package.load_entities() # Remove this to use Parquet

old_entity_path = os.path.join(pipeline.path, "old-entity.csv")
if os.path.exists(old_entity_path):
Expand All @@ -408,13 +406,12 @@ def dataset_create(
input_paths=input_paths,
specification_dir=None, # TBD: package should use this specification object
)
if False:
pqpackage.create_temp_table(input_paths)
pqpackage.load_facts(input_paths, cache_dir)
pqpackage.load_fact_resource(input_paths, cache_dir)
pqpackage.load_entities(input_paths, cache_dir, organisation_path)
pqpackage.pq_to_sqlite(output_path, cache_dir)
pqpackage.close_conn()
pqpackage.create_temp_table(input_paths)
pqpackage.load_facts(input_paths, cache_dir)
pqpackage.load_fact_resource(input_paths, cache_dir)
pqpackage.load_entities(input_paths, cache_dir, organisation_path)
pqpackage.pq_to_sqlite(output_path, cache_dir)
pqpackage.close_conn()


def dataset_dump(input_path, output_path):
Expand Down
3 changes: 0 additions & 3 deletions tests/acceptance/test_dataset_create.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@ def test_acceptance_dataset_create(
cache_path,
dataset_dir,
):
if True:
return

output_path = dataset_dir / f"{test_dataset}.sqlite3"

runner = CliRunner()
Expand Down

0 comments on commit beeaf1c

Please sign in to comment.