rw_pagerank_bucket_2_of_5.json

{
    "uuid": "d1689553-5bb8-4394-99eb-03f6d445f46b",
    "name": "rw_pagerank_bucket_2_of_5",
    "creation_date": "2024_01_27-01_35_30",
    "dataset_url": "s3://***REMOVED***/openlm/dcnlp/datasets/rw_pagerank_buckets/2_of_5/",
    "manifest_url": "s3://***REMOVED***/openlm/dcnlp/datasets/rw_pagerank_buckets/2_of_5/manifest.jsonl",
    "sources": [
        {
            "uuid": "21979427-e6ed-4404-9ee7-1f3b41239eeb",
            "name": "refinedweb-pagerank-bucket-2-of-5"
        }
    ],
    "tokenized": true,
    "tokenizer": "EleutherAI/gpt-neox-20b",
    "num_tokens": 48102705114,
    "size": 130754773688,
    "dcnlp_commit_hash": "295f17a9264efb6893e2dd55f4df59aaef3db5dc",
    "dcnlp_diff": "diff --git a/exp_data/datasets/tokenized/c4_original.json b/exp_data/datasets/tokenized/c4_original.json\nindex b2f9f97..9f0a4b8 100644\n--- a/exp_data/datasets/tokenized/c4_original.json\n+++ b/exp_data/datasets/tokenized/c4_original.json\n@@ -4,8 +4,8 @@\n     \"tokenized\": true,\n     \"num_tokens\": 174605508363,\n     \"size\": 1123288754203,\n-    \"dataset_url\": \"s3://***REMOVED***/original_c4/\",\n-    \"manifest_url\": \"s3://***REMOVED***/original_c4/manifest.jsonl\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/datasets/original_c4/\",\n+    \"manifest_url\": \"s3://***REMOVED***/openlm/dcnlp/datasets/original_c4/manifest.jsonl\",\n     \"dcnlp_commit_hash\": \"\",\n     \"dcnlp_diff\": \"\",\n     \"uuid\": \"7e0f5507-aa36-4d8c-9026-d049f885adf1\",\n@@ -13,4 +13,4 @@\n     \"tokenizer\": \"EleutherAI/gpt-neox-20b\",\n     \"data_key\": \"txt\",\n     \"sampling_yaml\": null\n-}\n\\ No newline at end of file\n+}\ndiff --git a/exp_data/datasets/tokenized/rw_original.json b/exp_data/datasets/tokenized/rw_original.json\nindex bed3824..d30b02d 100644\n--- a/exp_data/datasets/tokenized/rw_original.json\n+++ b/exp_data/datasets/tokenized/rw_original.json\n@@ -4,8 +4,8 @@\n     \"tokenized\": true,\n     \"num_tokens\": 579578773317,\n     \"size\": 1565888774322,\n-    \"dataset_url\": \"s3://***REMOVED***/refined_web_tokenized/\",\n-    \"manifest_url\": \"s3://***REMOVED***/refined_web_tokenized/manifest.jsonl\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/datasets/refined_web_tokenized/\",\n+    \"manifest_url\": \"s3://***REMOVED***/openlm/dcnlp/datasets/refined_web_tokenized/manifest.jsonl\",\n     \"dcnlp_commit_hash\": \"\",\n     \"dcnlp_diff\": \"\",\n     \"uuid\": \"7e0f5507-aa36-4d8c-9026-d049f885adf7\",\n@@ -13,4 +13,4 @@\n     \"tokenizer\": \"EleutherAI/gpt-neox-20b\",\n     \"data_key\": \"json.gz\",\n     \"sampling_yaml\": null\n-}\n\\ No newline at end of file\n+}\ndiff --git a/exp_data/datasets/untokenized/rpj_original.json b/exp_data/datasets/untokenized/rpj_original.json\nindex 817a094..d60f561 100644\n--- a/exp_data/datasets/untokenized/rpj_original.json\n+++ b/exp_data/datasets/untokenized/rpj_original.json\n@@ -2,7 +2,7 @@\n     \"uuid\": \"a49a6b1a-d357-475e-96a5-7a559ad927ef\",\n     \"name\": \"rpj_original\",\n     \"creation_date\": \"2024_01_05-10_38_45\",\n-    \"dataset_url\": \"s3://dcnlp-west/redpajama-real/\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/raw_datasets/redpajama_raw/\",\n     \"manifest_url\": null,\n     \"sources\": [],\n     \"tokenized\": false,\ndiff --git a/exp_data/datasets/untokenized/rpj_original_arxiv.json b/exp_data/datasets/untokenized/rpj_original_arxiv.json\nindex aea173a..21d27a8 100644\n--- a/exp_data/datasets/untokenized/rpj_original_arxiv.json\n+++ b/exp_data/datasets/untokenized/rpj_original_arxiv.json\n@@ -2,7 +2,7 @@\n     \"uuid\": \"c8b17a9b-6bd8-441a-8b9f-dbf486edf574\",\n     \"name\": \"rpj_original_arxiv\",\n     \"creation_date\": \"2023_12_31-14_21_45\",\n-    \"dataset_url\": \"s3://dcnlp-west/redpajama-real/arxiv/\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/raw_datasets/redpajama_raw/arxiv/\",\n     \"manifest_url\": null,\n     \"sources\": [],\n     \"tokenized\": false,\ndiff --git a/exp_data/datasets/untokenized/rpj_original_books.json b/exp_data/datasets/untokenized/rpj_original_books.json\nindex de40689..51f5c75 100644\n--- a/exp_data/datasets/untokenized/rpj_original_books.json\n+++ b/exp_data/datasets/untokenized/rpj_original_books.json\n@@ -2,7 +2,7 @@\n     \"uuid\": \"d017c1fe-c9df-4e06-aa8f-d92b1097283b\",\n     \"name\": \"rpj_original_books\",\n     \"creation_date\": \"2023_12_31-14_21_45\",\n-    \"dataset_url\": \"s3://dcnlp-west/redpajama-real/books_were_too_long_for_vaishaal_to_read/\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/raw_datasets/redpajama_raw/books_were_too_long_for_vaishaal_to_read/\",\n     \"manifest_url\": null,\n     \"sources\": [],\n     \"tokenized\": false,\ndiff --git a/exp_data/datasets/untokenized/rpj_original_cc.json b/exp_data/datasets/untokenized/rpj_original_cc.json\nindex 4a322df..e538171 100644\n--- a/exp_data/datasets/untokenized/rpj_original_cc.json\n+++ b/exp_data/datasets/untokenized/rpj_original_cc.json\n@@ -2,7 +2,7 @@\n     \"uuid\": \"15701e36-c0bb-4bfa-bf52-d3419dbbd8a1\",\n     \"name\": \"rpj_original_cc\",\n     \"creation_date\": \"2024_01_05-10_38_45\",\n-    \"dataset_url\": \"s3://dcnlp-west/redpajama-real/common_crawl/\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/raw_datasets/redpajama_raw/common_crawl/\",\n     \"manifest_url\": null,\n     \"sources\": [],\n     \"tokenized\": false,\ndiff --git a/exp_data/datasets/untokenized/rpj_original_github.json b/exp_data/datasets/untokenized/rpj_original_github.json\nindex 1380c00..d7546f7 100644\n--- a/exp_data/datasets/untokenized/rpj_original_github.json\n+++ b/exp_data/datasets/untokenized/rpj_original_github.json\n@@ -2,7 +2,7 @@\n     \"uuid\": \"edd67f24-49ae-4915-8c3a-dd4bcc62b9d8\",\n     \"name\": \"rpj_original_github\",\n     \"creation_date\": \"2023_12_31-14_21_45\",\n-    \"dataset_url\": \"s3://dcnlp-west/redpajama-real/github/\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/raw_datasets/redpajama_raw/github/\",\n     \"manifest_url\": null,\n     \"sources\": [],\n     \"tokenized\": false,\ndiff --git a/exp_data/datasets/untokenized/rpj_original_non_CC.json b/exp_data/datasets/untokenized/rpj_original_non_CC.json\nindex 181fbe5..bade67c 100644\n--- a/exp_data/datasets/untokenized/rpj_original_non_CC.json\n+++ b/exp_data/datasets/untokenized/rpj_original_non_CC.json\n@@ -2,7 +2,7 @@\n     \"uuid\": \"807c9277-7b10-4133-882d-09e22369587b\",\n     \"name\": \"rpj_original_non_CC\",\n     \"creation_date\": \"2023_12_31-14_21_45\",\n-    \"dataset_url\": \"s3://dcnlp-west/redpajama-real/\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/raw_datasets/redpajama_raw/\",\n     \"manifest_url\": null,\n     \"sources\": [],\n     \"tokenized\": false,\ndiff --git a/exp_data/datasets/untokenized/rpj_original_stackexchange.json b/exp_data/datasets/untokenized/rpj_original_stackexchange.json\nindex 12290b1..f337d4c 100644\n--- a/exp_data/datasets/untokenized/rpj_original_stackexchange.json\n+++ b/exp_data/datasets/untokenized/rpj_original_stackexchange.json\n@@ -2,7 +2,7 @@\n     \"uuid\": \"3b25b18c-e724-4071-8c7a-d69c5e1aaeac\",\n     \"name\": \"rpj_original_stackexchange\",\n     \"creation_date\": \"2023_12_31-14_21_45\",\n-    \"dataset_url\": \"s3://dcnlp-west/redpajama-real/stackexchange/\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/raw_datasets/redpajama_raw/stackexchange/\",\n     \"manifest_url\": null,\n     \"sources\": [],\n     \"tokenized\": false,\ndiff --git a/exp_data/datasets/untokenized/rpj_original_wiki.json b/exp_data/datasets/untokenized/rpj_original_wiki.json\nindex d98f66b..b7f70b0 100644\n--- a/exp_data/datasets/untokenized/rpj_original_wiki.json\n+++ b/exp_data/datasets/untokenized/rpj_original_wiki.json\n@@ -2,7 +2,7 @@\n     \"uuid\": \"050bc436-8d61-4d73-b931-0306a4b26727\",\n     \"name\": \"rpj_original_wiki\",\n     \"creation_date\": \"2023_12_31-14_21_45\",\n-    \"dataset_url\": \"s3://dcnlp-west/redpajama-real/wiki/\",\n+    \"dataset_url\": \"s3://***REMOVED***/openlm/dcnlp/raw_datasets/redpajama_raw/wiki/\",\n     \"manifest_url\": null,\n     \"sources\": [],\n     \"tokenized\": false,\ndiff --git a/ray_processing/__init__.py b/ray_processing/__init__.py\nindex 5e1b41d..014c770 100644\n--- a/ray_processing/__init__.py\n+++ b/ray_processing/__init__.py\n@@ -1,4 +1,4 @@\n-from dedup_jsonl import dedup_jsonl\n+from ray_processing.dedup_jsonl import dedup_jsonl\n from baselines.core.constants import GLOBAL_FUNCTIONS\n \n-GLOBAL_FUNCTIONS['exact_dedup'] = dedup_jsonl\n\\ No newline at end of file\n+GLOBAL_FUNCTIONS['exact_dedup'] = dedup_jsonl\ndiff --git a/ray_processing/cluster_tri_tokenize_shuffle.yaml b/ray_processing/cluster_tri_tokenize_shuffle.yaml\nindex 4979458..29564ed 100644\n--- a/ray_processing/cluster_tri_tokenize_shuffle.yaml\n+++ b/ray_processing/cluster_tri_tokenize_shuffle.yaml\n@@ -30,7 +30,7 @@ provider:\n \n # Mount local copy of DCNLP instead of cloning\n file_mounts: {\n-    \"/home/ubuntu/dcnlp\": \"../\",\n+    \"/home/ubuntu/dcnlp\": \"./\",\n }\n \n # Add any paths you don't want to copy from your dcnlp repo.\ndiff --git a/ray_processing/tokenize_shuffle.py b/ray_processing/tokenize_shuffle.py\nindex ba2ac32..14d4125 100644\n--- a/ray_processing/tokenize_shuffle.py\n+++ b/ray_processing/tokenize_shuffle.py\n@@ -53,7 +53,9 @@ if __name__ == \"__main__\":\n         assert all(s is not None for s in source_refs), \"Not all source reference jsons could be found.\"\n \n     # Collect args for tokenization and pass them into tokenize_shuffle\n-    tokenize_shuffle_args = [str(i) for k,v in vars(args).items() for i in [f\"--{k}\", v] if k not in DCNLP_ARGS and v]\n+    tokenize_shuffle_args = [str(i) for k,v in vars(args).items() for i in [f\"--{k}\", v] if k not in DCNLP_ARGS and k != \"do_sample\" and v]\n+    if args.do_sample:\n+        tokenize_shuffle_args += [\"--do_sample\"]\n     tokenize_shuffle.main(tokenize_shuffle_args)\n \n     dataset_json = generate_tokenized_dataset_json(args, source_refs)\ndiff --git a/training/configs/11m_1x.json b/training/configs/11m_1x.json\nindex 3cd6916..4455d89 100644\n--- a/training/configs/11m_1x.json\n+++ b/training/configs/11m_1x.json\n@@ -18,4 +18,4 @@\n         \"--fsdp-limit-all-gathers\"\n     ],\n     \"chinchilla_multiplier\": 1\n-}\n\\ No newline at end of file\n+}\ndiff --git a/training/configs/154m_1x.json b/training/configs/154m_1x.json\nindex 6ee80b2..0857fbe 100644\n--- a/training/configs/154m_1x.json\n+++ b/training/configs/154m_1x.json\n@@ -8,7 +8,7 @@\n     \"wd\": 0.033,\n     \"cd\": 3e-05,\n     \"global_bs\": 512,\n-    \"acc\": 8,\n+    \"acc\": 2,\n     \"qk_norm\": true,\n     \"z_loss\": 1e-4,\n     \"grad_checkpointing\": false,\n@@ -18,4 +18,4 @@\n         \"--fsdp-limit-all-gathers\"\n     ],\n     \"chinchilla_multiplier\": 1\n-}\n\\ No newline at end of file\n+}\ndiff --git a/training/configs/1b_1x.json b/training/configs/1b_1x.json\nindex bd0a40b..45b4656 100644\n--- a/training/configs/1b_1x.json\n+++ b/training/configs/1b_1x.json\n@@ -8,7 +8,7 @@\n     \"wd\": 0.033,\n     \"cd\": 3e-5,\n     \"global_bs\": 256,\n-    \"acc\": 2,\n+    \"acc\": 1,\n     \"qk_norm\": true,\n     \"z_loss\": 1e-4,\n     \"grad_checkpointing\": false,\n@@ -18,4 +18,4 @@\n         \"--fsdp-limit-all-gathers\"\n     ],\n     \"chinchilla_multiplier\": 1\n-}\n\\ No newline at end of file\n+}\ndiff --git a/training/configs/3b_1x.json b/training/configs/3b_1x.json\nindex d77a4d4..2e9e15b 100644\n--- a/training/configs/3b_1x.json\n+++ b/training/configs/3b_1x.json\n@@ -8,7 +8,7 @@\n     \"wd\": 0.33,\n     \"cd\": 3e-05,\n     \"global_bs\": 2048,\n-    \"acc\": 2,\n+    \"acc\": 4,\n     \"qk_norm\": true,\n     \"z_loss\": 1e-4,\n     \"grad_checkpointing\": false,\ndiff --git a/training/configs/411m_1x.json b/training/configs/411m_1x.json\nindex 85a7d1e..8094598 100644\n--- a/training/configs/411m_1x.json\n+++ b/training/configs/411m_1x.json\n@@ -8,7 +8,7 @@\n     \"wd\": 0.033,\n     \"cd\": 3e-05,\n     \"global_bs\": 512,\n-    \"acc\": 8,\n+    \"acc\": 2,\n     \"qk_norm\": true,\n     \"z_loss\": 1e-4,\n     \"grad_checkpointing\": false,\n@@ -18,4 +18,4 @@\n         \"--fsdp-limit-all-gathers\"\n     ],\n     \"chinchilla_multiplier\": 1\n-}\n\\ No newline at end of file\n+}\ndiff --git a/training/file_utils.py b/training/file_utils.py\nindex 8655e22..fe81304 100644\n--- a/training/file_utils.py\n+++ b/training/file_utils.py\n@@ -221,7 +221,7 @@ def download_val_data(name, root=Path(__file__).parent / f\"eval_data/\", skip_dow\n     if name in DOWNSTREAM_SHARD_HASHES:\n         # case where request a special downstream shard for eval, populate accordingly\n \n-        tasks = load_heavy_yaml()\n+        tasks = load_ppl_yaml()\n         category = tasks[name][\"dataset_uri\"].split(\"/\")[1]\n \n         cloud_checkpoints[name] = {\ndiff --git a/training/params.py b/training/params.py\nindex 01d49bf..b1f0f10 100644\n--- a/training/params.py\n+++ b/training/params.py\n@@ -7,7 +7,7 @@ import torch.distributed as dist\n from open_lm.distributed import world_info_from_env\n \n from training.hyperparameters import available_scales\n-from training.file_utils import download_val_data, load_heavy_yaml, tok_mult_paths\n+from training.file_utils import download_val_data, load_ppl_yaml, tok_mult_paths\n \n \n def parse_dcnlp_args():\n@@ -271,7 +271,7 @@ def get_open_lm_args(args, hparams, dr):\n     paloma_val_data = download_val_data(\"paloma_val\", skip_download=local_rank != 0)\n \n     if args.downstream_eval:\n-        tasks = load_heavy_yaml()\n+        tasks = load_ppl_yaml()\n         downstream_datas = [download_val_data(task_name, skip_download=local_rank != 0) for task_name in tasks]\n \n         open_lm_args.extend(\n@@ -317,7 +317,7 @@ def get_open_lm_args(args, hparams, dr):\n                 \"--val-data\",\n                 openlm_val_data,\n                 c4_val_data,\n-                paloma_val_data,\n+                # paloma_val_data,\n                 \"--val-frequency\",\n                 f\"{args.val_frequency}\",\n                 \"--val-data-key\",\ndiff --git a/training/train.py b/training/train.py\nindex 3f292f6..1cb3df5 100644\n--- a/training/train.py\n+++ b/training/train.py\n@@ -150,7 +150,9 @@ if __name__ == \"__main__\":\n             json.dump(asdict(model), f, indent=4)\n \n         if args.remote_sync:\n-            with fs.open(os.path.join(exp_root, f\"{name}.json\"), \"w\") as f:\n+            remote_model_path = os.path.join(exp_root, f\"{name}.json\")\n+            print(f\"Writing model reference to remote path: {remote_model_path}\")\n+            with fs.open(remote_model_path, \"w\") as f:\n                 json.dump(asdict(model), f, indent=4)\n \n         # clean up as needed\ndiff --git a/training/train_scripts/docker/Dockerfile_update b/training/train_scripts/docker/Dockerfile_update\nindex b46252b..18e49d8 100644\n--- a/training/train_scripts/docker/Dockerfile_update\n+++ b/training/train_scripts/docker/Dockerfile_update\n@@ -8,7 +8,7 @@ COPY . /opt/ml/code/\n \n # RUN pip install -e /opt/ml/code/\n \n-# # Prevent sagemaker from installing requirements again.\n+# Prevent sagemaker from installing requirements again.\n RUN rm /opt/ml/code/requirements.txt\n \n ENV SAGEMAKER_PROGRAM training/train.py\ndiff --git a/training/train_scripts/train_sagemaker.py b/training/train_scripts/train_sagemaker.py\nindex 0d29c07..cca62e3 100644\n--- a/training/train_scripts/train_sagemaker.py\n+++ b/training/train_scripts/train_sagemaker.py\n@@ -85,6 +85,7 @@ def main():\n     parser.add_argument(\"--scale\", required=True)\n     parser.add_argument(\"--data-config\", required=True)\n     parser.add_argument(\"--remote-sync\", required=True, help=\"S3 path to sync to\")\n+    parser.add_argument(\"--chinchilla-multiplier\", required=False, type=float)\n \n     # Docker / AWS args\n     parser.add_argument(\"--docker-dir\", type=Path, default=Path(__file__).parent / \"docker\")\n@@ -165,6 +166,8 @@ def main_after_setup_move(args):\n         \"logs\": f\"{checkpoint_local_path}/{job_name}\",\n         \"report-to-wandb\": \"\",\n     }\n+    if args.chinchilla_multiplier:\n+        train_args[\"chinchilla-multiplier\"] = args.chinchilla_multiplier\n \n     estimator = PyTorch(\n         entry_point=\"training/train.py\",",
    "data_key": "json.gz",
    "sampling_yaml": null
}