settings-default.toml

[general]
log_file = "vat_abcd_crawler.log"
debug = false

[debug]
dataset_start = 0
dataset_limit = 3

[abcd]
fields_file = "abcd-fields.json"
landing_page_field = "/DataSets/DataSet/Metadata/Description/Representation/URI"
storage_dir = "raw_data"

[pangaea]
search_url = "https://elasticsearch.gfbio.dev/dataportal-gfbio/_search"
scroll_url = "https://elasticsearch.gfbio.dev/_search/scroll"

[terminology_service]
landingpage_url = "https://terminologies.gfbio.org/tools/landingpages/landingpage.php"

[database]
host = "localhost"
port = 5432
tls = true
database = ""
user = ""
password = ""
schema = ""
dataset_table = "abcd_datasets"
temp_dataset_table = "abcd_datasets_temp"
surrogate_key_column = "surrogate_key"
dataset_id_column = "dataset_id"
dataset_path_column = "dataset_path"
dataset_landing_page_column = "dataset_landing_page"
dataset_provider_column = "dataset_provider"
unit_table = "abcd_units"
temp_unit_table = "abcd_units_temp"
listing_view = "dataset_listing"
unit_indexed_columns = [
    "/DataSets/DataSet/Units/Unit/Gathering/SiteCoordinateSets/SiteCoordinates/CoordinatesLatLong/LongitudeDecimal",
    "/DataSets/DataSet/Units/Unit/Gathering/SiteCoordinateSets/SiteCoordinates/CoordinatesLatLong/LatitudeDecimal",
]