From fb81854aa4b4007eb4db59fd41edabe0d67e7cf0 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Mon, 19 Dec 2022 15:27:21 -0500 Subject: [PATCH 01/15] collapse pco sub-orgs' briefingt/qpnotes records under pco --- bin/migrate/migrate_briefingt_2022_12.py | 36 ++++++++++++++++++++++++ bin/migrate/migrate_qpnotes_2022_12.py | 36 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 bin/migrate/migrate_briefingt_2022_12.py create mode 100644 bin/migrate/migrate_qpnotes_2022_12.py diff --git a/bin/migrate/migrate_briefingt_2022_12.py b/bin/migrate/migrate_briefingt_2022_12.py new file mode 100644 index 000000000..9729f81d6 --- /dev/null +++ b/bin/migrate/migrate_briefingt_2022_12.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all briefingt records +for PCO sub-organizations into PCO organization +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vmp', + 'pqcc-pcprc', + 'ql-lq', + 'srp-rsp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +for line in in_csv: + if line['owner_org'] in sub_orgs: + line['owner_org'] = PCO['owner_org'] + line['owner_org_title'] = PCO['owner_org_title'] + out_csv.writerow(line) diff --git a/bin/migrate/migrate_qpnotes_2022_12.py b/bin/migrate/migrate_qpnotes_2022_12.py new file mode 100644 index 000000000..39f1a0587 --- /dev/null +++ b/bin/migrate/migrate_qpnotes_2022_12.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all qpnotes records +for PCO sub-organizations into PCO organization +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vmp', + 'pqcc-pcprc', + 'ql-lq', + 'srp-rsp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +for line in in_csv: + if line['owner_org'] in sub_orgs: + line['owner_org'] = PCO['owner_org'] + line['owner_org_title'] = PCO['owner_org_title'] + out_csv.writerow(line) From 26db8dabb7eff2181ec5e551e78631414fb6e2f2 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Tue, 20 Dec 2022 10:12:11 -0500 Subject: [PATCH 02/15] update pco-sub org name change to pkcc-pcprc --- bin/migrate/migrate_briefingt_2022_12.py | 2 +- bin/migrate/migrate_qpnotes_2022_12.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/migrate/migrate_briefingt_2022_12.py b/bin/migrate/migrate_briefingt_2022_12.py index 9729f81d6..299b5d6b4 100644 --- a/bin/migrate/migrate_briefingt_2022_12.py +++ b/bin/migrate/migrate_briefingt_2022_12.py @@ -16,7 +16,7 @@ 'mdi-mid', 'miga-maig', 'dpm-vmp', - 'pqcc-pcprc', + 'pkcc-pcprc', 'ql-lq', 'srp-rsp', ] diff --git a/bin/migrate/migrate_qpnotes_2022_12.py b/bin/migrate/migrate_qpnotes_2022_12.py index 39f1a0587..aa40cc96c 100644 --- a/bin/migrate/migrate_qpnotes_2022_12.py +++ b/bin/migrate/migrate_qpnotes_2022_12.py @@ -16,7 +16,7 @@ 'mdi-mid', 'miga-maig', 'dpm-vmp', - 'pqcc-pcprc', + 'pkcc-pcprc', 'ql-lq', 'srp-rsp', ] From 3601faef39f4e9a25256262f9d42fc985aeb6e28 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Tue, 20 Dec 2022 10:49:14 -0500 Subject: [PATCH 03/15] minor improvement to pco sub-orgs migration script --- bin/migrate/migrate_briefingt_2022_12.py | 5 ++--- bin/migrate/migrate_qpnotes_2022_12.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/migrate/migrate_briefingt_2022_12.py b/bin/migrate/migrate_briefingt_2022_12.py index 299b5d6b4..8962abb77 100644 --- a/bin/migrate/migrate_briefingt_2022_12.py +++ b/bin/migrate/migrate_briefingt_2022_12.py @@ -15,7 +15,7 @@ 'iga-aig', 'mdi-mid', 'miga-maig', - 'dpm-vmp', + 'dpm-vpm', 'pkcc-pcprc', 'ql-lq', 'srp-rsp', @@ -31,6 +31,5 @@ for line in in_csv: if line['owner_org'] in sub_orgs: - line['owner_org'] = PCO['owner_org'] - line['owner_org_title'] = PCO['owner_org_title'] + line.update(PCO) out_csv.writerow(line) diff --git a/bin/migrate/migrate_qpnotes_2022_12.py b/bin/migrate/migrate_qpnotes_2022_12.py index aa40cc96c..78efa27a8 100644 --- a/bin/migrate/migrate_qpnotes_2022_12.py +++ b/bin/migrate/migrate_qpnotes_2022_12.py @@ -15,7 +15,7 @@ 'iga-aig', 'mdi-mid', 'miga-maig', - 'dpm-vmp', + 'dpm-vpm', 'pkcc-pcprc', 'ql-lq', 'srp-rsp', @@ -31,6 +31,5 @@ for line in in_csv: if line['owner_org'] in sub_orgs: - line['owner_org'] = PCO['owner_org'] - line['owner_org_title'] = PCO['owner_org_title'] + line.update(PCO) out_csv.writerow(line) From e2f571988ff7d966a26aa78270aa87b4f4ee5a51 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Tue, 20 Dec 2022 10:50:36 -0500 Subject: [PATCH 04/15] add pco sub-orgs migration scripts for travelq and hospitalityq --- bin/migrate/migrate_hospitalityq_2022_12.py | 35 +++++++++++++++++++++ bin/migrate/migrate_travelq_2022_12.py | 35 +++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 bin/migrate/migrate_hospitalityq_2022_12.py create mode 100644 bin/migrate/migrate_travelq_2022_12.py diff --git a/bin/migrate/migrate_hospitalityq_2022_12.py b/bin/migrate/migrate_hospitalityq_2022_12.py new file mode 100644 index 000000000..56de00cdd --- /dev/null +++ b/bin/migrate/migrate_hospitalityq_2022_12.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all hospitalityq records +for PCO sub-organizations into PCO organization +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vpm', + 'pkcc-pcprc', + 'ql-lq', + 'srp-rsp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +for line in in_csv: + if line['owner_org'] in sub_orgs: + line.update(PCO) + out_csv.writerow(line) diff --git a/bin/migrate/migrate_travelq_2022_12.py b/bin/migrate/migrate_travelq_2022_12.py new file mode 100644 index 000000000..7cf013fe5 --- /dev/null +++ b/bin/migrate/migrate_travelq_2022_12.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all travelq records +for PCO sub-organizations into PCO organization +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vpm', + 'pkcc-pcprc', + 'ql-lq', + 'srp-rsp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +for line in in_csv: + if line['owner_org'] in sub_orgs: + line.update(PCO) + out_csv.writerow(line) From 8205820e8bca16afc8898559a07055c25dfafaef Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Fri, 23 Dec 2022 10:17:25 -0500 Subject: [PATCH 05/15] migration script to collapse pco sub-orgs for travelq-nil --- bin/migrate/migrate_travelq_nil_2022_12.py | 53 ++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 bin/migrate/migrate_travelq_nil_2022_12.py diff --git a/bin/migrate/migrate_travelq_nil_2022_12.py b/bin/migrate/migrate_travelq_nil_2022_12.py new file mode 100644 index 000000000..d1bd17e77 --- /dev/null +++ b/bin/migrate/migrate_travelq_nil_2022_12.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all travelq-nil records +for PCO sub-organizations into PCO organization. +Duplicate records from sub-orgnizations for (year,month) +will be ignored +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vpm', + 'pkcc-pcprc', + 'ql-lq', + 'srp-rsp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +data = [] + +def report_exists(year, month, owner_org): + for row in data: + if row['year'] == year and \ + row['month'] == month and \ + row['owner_org'] == owner_org: + return True + return False + +for line in in_csv: + if line['owner_org'] in sub_orgs: + if not report_exists(line['year'], line['month'], PCO['owner_org']): + line.update(PCO) + data.append(line) + else: + data.append(line) + +for line in data: + out_csv.writerow(line) From da9f9801df11ce118b26908019d42b1af01f4258 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Fri, 23 Dec 2022 11:58:11 -0500 Subject: [PATCH 06/15] migration script to collapse pco sub-orgs for hospitalityq-nil --- .../migrate_hospitalityq_nil_2022_12.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 bin/migrate/migrate_hospitalityq_nil_2022_12.py diff --git a/bin/migrate/migrate_hospitalityq_nil_2022_12.py b/bin/migrate/migrate_hospitalityq_nil_2022_12.py new file mode 100644 index 000000000..108c9470e --- /dev/null +++ b/bin/migrate/migrate_hospitalityq_nil_2022_12.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all hospitalityq-nil records +for PCO sub-organizations into PCO organization. +Duplicate records from sub-orgnizations for (year,month) +will be ignored +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vpm', + 'pkcc-pcprc', + 'ql-lq', + 'srp-rsp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +data = [] + +def report_exists(year, month, owner_org): + for row in data: + if row['year'] == year and \ + row['month'] == month and \ + row['owner_org'] == owner_org: + return True + return False + +for line in in_csv: + if line['owner_org'] in sub_orgs: + if not report_exists(line['year'], line['month'], PCO['owner_org']): + line.update(PCO) + data.append(line) + else: + data.append(line) + +for line in data: + out_csv.writerow(line) From eebf379cf29f348f8aa7ef68d44ef8b9ee74e837 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Wed, 1 Mar 2023 13:15:06 -0500 Subject: [PATCH 07/15] add more organizations to migration scripts for pco --- bin/migrate/migrate_briefingt_2022_12.py | 5 +++++ bin/migrate/migrate_hospitalityq_2022_12.py | 5 +++++ bin/migrate/migrate_hospitalityq_nil_2022_12.py | 5 +++++ bin/migrate/migrate_qpnotes_2022_12.py | 5 +++++ bin/migrate/migrate_travelq_2022_12.py | 5 +++++ bin/migrate/migrate_travelq_nil_2022_12.py | 5 +++++ 6 files changed, 30 insertions(+) diff --git a/bin/migrate/migrate_briefingt_2022_12.py b/bin/migrate/migrate_briefingt_2022_12.py index 8962abb77..d0bfcad19 100644 --- a/bin/migrate/migrate_briefingt_2022_12.py +++ b/bin/migrate/migrate_briefingt_2022_12.py @@ -19,6 +19,11 @@ 'pkcc-pcprc', 'ql-lq', 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_hospitalityq_2022_12.py b/bin/migrate/migrate_hospitalityq_2022_12.py index 56de00cdd..54889985e 100644 --- a/bin/migrate/migrate_hospitalityq_2022_12.py +++ b/bin/migrate/migrate_hospitalityq_2022_12.py @@ -19,6 +19,11 @@ 'pkcc-pcprc', 'ql-lq', 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_hospitalityq_nil_2022_12.py b/bin/migrate/migrate_hospitalityq_nil_2022_12.py index 108c9470e..00bc20d5e 100644 --- a/bin/migrate/migrate_hospitalityq_nil_2022_12.py +++ b/bin/migrate/migrate_hospitalityq_nil_2022_12.py @@ -21,6 +21,11 @@ 'pkcc-pcprc', 'ql-lq', 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_qpnotes_2022_12.py b/bin/migrate/migrate_qpnotes_2022_12.py index 78efa27a8..0a19a00f0 100644 --- a/bin/migrate/migrate_qpnotes_2022_12.py +++ b/bin/migrate/migrate_qpnotes_2022_12.py @@ -19,6 +19,11 @@ 'pkcc-pcprc', 'ql-lq', 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_travelq_2022_12.py b/bin/migrate/migrate_travelq_2022_12.py index 7cf013fe5..f9ae73ad1 100644 --- a/bin/migrate/migrate_travelq_2022_12.py +++ b/bin/migrate/migrate_travelq_2022_12.py @@ -19,6 +19,11 @@ 'pkcc-pcprc', 'ql-lq', 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_travelq_nil_2022_12.py b/bin/migrate/migrate_travelq_nil_2022_12.py index d1bd17e77..fadc91f58 100644 --- a/bin/migrate/migrate_travelq_nil_2022_12.py +++ b/bin/migrate/migrate_travelq_nil_2022_12.py @@ -21,6 +21,11 @@ 'pkcc-pcprc', 'ql-lq', 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } From a7cb68ac3caa832b766c3905ba24f299d53fa149 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Wed, 1 Mar 2023 13:57:26 -0500 Subject: [PATCH 08/15] add migration scripts for contracts, travela for pco merge --- bin/migrate/migrate_contracts_2022_12.py | 40 ++++++++++++++++++++++++ bin/migrate/migrate_travela_2022_12.py | 40 ++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 bin/migrate/migrate_contracts_2022_12.py create mode 100644 bin/migrate/migrate_travela_2022_12.py diff --git a/bin/migrate/migrate_contracts_2022_12.py b/bin/migrate/migrate_contracts_2022_12.py new file mode 100644 index 000000000..bbe736b3b --- /dev/null +++ b/bin/migrate/migrate_contracts_2022_12.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all contracts records +for PCO sub-organizations into PCO organization +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vpm', + 'pkcc-pcprc', + 'ql-lq', + 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +for line in in_csv: + if line['owner_org'] in sub_orgs: + line.update(PCO) + out_csv.writerow(line) diff --git a/bin/migrate/migrate_travela_2022_12.py b/bin/migrate/migrate_travela_2022_12.py new file mode 100644 index 000000000..d5636d5e4 --- /dev/null +++ b/bin/migrate/migrate_travela_2022_12.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all travela records +for PCO sub-organizations into PCO organization +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vpm', + 'pkcc-pcprc', + 'ql-lq', + 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +for line in in_csv: + if line['owner_org'] in sub_orgs: + line.update(PCO) + out_csv.writerow(line) From ead9e2121ade8d6d5e4b70086e70ef27d7c7bcc8 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Wed, 1 Mar 2023 15:57:46 -0500 Subject: [PATCH 09/15] add migration script for contracts-nil for pco-merge --- bin/migrate/migrate_contracts_nil_2022_12.py | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 bin/migrate/migrate_contracts_nil_2022_12.py diff --git a/bin/migrate/migrate_contracts_nil_2022_12.py b/bin/migrate/migrate_contracts_nil_2022_12.py new file mode 100644 index 000000000..9900170ac --- /dev/null +++ b/bin/migrate/migrate_contracts_nil_2022_12.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# coding=utf-8 + +""" +migration script to copy all contracts-nil records +for PCO sub-organizations into PCO organization. +Duplicate records from sub-orgnizations for +reporting_period will be ignored +""" + +import unicodecsv +import sys +import codecs + + +sub_orgs = [ 'ghl-lgc', + 'iga-aig', + 'mdi-mid', + 'miga-maig', + 'dpm-vpm', + 'pkcc-pcprc', + 'ql-lq', + 'srp-rsp', + 'nsira-ossnr', + 'ocsec-bccst', + 'snsicp-scpssnr', + 'sirc-csars', + 'jfpc-cfp', + ] +PCO = { 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +out_csv.writeheader() + +data = [] + +def report_exists(reporting_period, owner_org): + for row in data: + if row['reporting_period'] == reporting_period and \ + row['owner_org'] == owner_org: + return True + return False + +for line in in_csv: + if line['owner_org'] in sub_orgs: + if not report_exists(line['reporting_period'], PCO['owner_org']): + line.update(PCO) + data.append(line) + else: + data.append(line) + +for line in data: + out_csv.writerow(line) From 2063c9888756b6c85f360f984a76b209cac577c4 Mon Sep 17 00:00:00 2001 From: RabiaSajjad Date: Thu, 30 Jan 2025 14:23:09 -0500 Subject: [PATCH 10/15] update migration scripts for pco sub-orgs --- ...022_12.py => migrate_briefingt_2025_02.py} | 23 +++++------ ...12.py => migrate_briefingt_nil_2025_12.py} | 25 +++++------- bin/migrate/migrate_contracts_2022_12.py | 40 ------------------- ..._12.py => migrate_hospitalityq_2025_02.py} | 23 +++++------ ...py => migrate_hospitalityq_nil_2025_02.py} | 23 +++++------ ..._2022_12.py => migrate_qpnotes_2025_02.py} | 23 +++++------ ..._2022_12.py => migrate_travela_2025_02.py} | 23 +++++------ ..._2022_12.py => migrate_travelq_2025_02.py} | 23 +++++------ ...2_12.py => migrate_travelq_nil_2025_02.py} | 23 +++++------ 9 files changed, 73 insertions(+), 153 deletions(-) rename bin/migrate/{migrate_briefingt_2022_12.py => migrate_briefingt_2025_02.py} (64%) rename bin/migrate/{migrate_contracts_nil_2022_12.py => migrate_briefingt_nil_2025_12.py} (72%) delete mode 100644 bin/migrate/migrate_contracts_2022_12.py rename bin/migrate/{migrate_hospitalityq_2022_12.py => migrate_hospitalityq_2025_02.py} (64%) rename bin/migrate/{migrate_hospitalityq_nil_2022_12.py => migrate_hospitalityq_nil_2025_02.py} (76%) rename bin/migrate/{migrate_qpnotes_2022_12.py => migrate_qpnotes_2025_02.py} (64%) rename bin/migrate/{migrate_travela_2022_12.py => migrate_travela_2025_02.py} (64%) rename bin/migrate/{migrate_travelq_2022_12.py => migrate_travelq_2025_02.py} (64%) rename bin/migrate/{migrate_travelq_nil_2022_12.py => migrate_travelq_nil_2025_02.py} (76%) diff --git a/bin/migrate/migrate_briefingt_2022_12.py b/bin/migrate/migrate_briefingt_2025_02.py similarity index 64% rename from bin/migrate/migrate_briefingt_2022_12.py rename to bin/migrate/migrate_briefingt_2025_02.py index d0bfcad19..dfe89ab1e 100644 --- a/bin/migrate/migrate_briefingt_2022_12.py +++ b/bin/migrate/migrate_briefingt_2025_02.py @@ -11,20 +11,15 @@ import codecs -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] +sub_orgs = [ + 'dpm-vpm', + 'iga-aig', + 'miga-maig', + 'ghl-lgc', + 'mdi-mid', + 'ql-lq', + 'srp-rsp', + ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_contracts_nil_2022_12.py b/bin/migrate/migrate_briefingt_nil_2025_12.py similarity index 72% rename from bin/migrate/migrate_contracts_nil_2022_12.py rename to bin/migrate/migrate_briefingt_nil_2025_12.py index 9900170ac..940f1da7c 100644 --- a/bin/migrate/migrate_contracts_nil_2022_12.py +++ b/bin/migrate/migrate_briefingt_nil_2025_12.py @@ -2,7 +2,7 @@ # coding=utf-8 """ -migration script to copy all contracts-nil records +migration script to copy all briefingt-nil records for PCO sub-organizations into PCO organization. Duplicate records from sub-orgnizations for reporting_period will be ignored @@ -13,20 +13,15 @@ import codecs -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] +sub_orgs = [ + 'dpm-vpm', + 'iga-aig', + 'miga-maig', + 'ghl-lgc', + 'mdi-mid', + 'ql-lq', + 'srp-rsp', + ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_contracts_2022_12.py b/bin/migrate/migrate_contracts_2022_12.py deleted file mode 100644 index bbe736b3b..000000000 --- a/bin/migrate/migrate_contracts_2022_12.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 - -""" -migration script to copy all contracts records -for PCO sub-organizations into PCO organization -""" - -import unicodecsv -import sys -import codecs - - -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } - -assert sys.stdin.read(3) == codecs.BOM_UTF8 - -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') -out_csv.writeheader() - -for line in in_csv: - if line['owner_org'] in sub_orgs: - line.update(PCO) - out_csv.writerow(line) diff --git a/bin/migrate/migrate_hospitalityq_2022_12.py b/bin/migrate/migrate_hospitalityq_2025_02.py similarity index 64% rename from bin/migrate/migrate_hospitalityq_2022_12.py rename to bin/migrate/migrate_hospitalityq_2025_02.py index 54889985e..31062c2c5 100644 --- a/bin/migrate/migrate_hospitalityq_2022_12.py +++ b/bin/migrate/migrate_hospitalityq_2025_02.py @@ -11,20 +11,15 @@ import codecs -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] +sub_orgs = [ + 'dpm-vpm', + 'iga-aig', + 'miga-maig', + 'ghl-lgc', + 'mdi-mid', + 'ql-lq', + 'srp-rsp', + ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_hospitalityq_nil_2022_12.py b/bin/migrate/migrate_hospitalityq_nil_2025_02.py similarity index 76% rename from bin/migrate/migrate_hospitalityq_nil_2022_12.py rename to bin/migrate/migrate_hospitalityq_nil_2025_02.py index 00bc20d5e..1355fd765 100644 --- a/bin/migrate/migrate_hospitalityq_nil_2022_12.py +++ b/bin/migrate/migrate_hospitalityq_nil_2025_02.py @@ -13,20 +13,15 @@ import codecs -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] +sub_orgs = [ + 'dpm-vpm', + 'iga-aig', + 'miga-maig', + 'ghl-lgc', + 'mdi-mid', + 'ql-lq', + 'srp-rsp', + ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_qpnotes_2022_12.py b/bin/migrate/migrate_qpnotes_2025_02.py similarity index 64% rename from bin/migrate/migrate_qpnotes_2022_12.py rename to bin/migrate/migrate_qpnotes_2025_02.py index 0a19a00f0..0dc45eb9b 100644 --- a/bin/migrate/migrate_qpnotes_2022_12.py +++ b/bin/migrate/migrate_qpnotes_2025_02.py @@ -11,20 +11,15 @@ import codecs -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] +sub_orgs = [ + 'dpm-vpm', + 'iga-aig', + 'miga-maig', + 'ghl-lgc', + 'mdi-mid', + 'ql-lq', + 'srp-rsp', + ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_travela_2022_12.py b/bin/migrate/migrate_travela_2025_02.py similarity index 64% rename from bin/migrate/migrate_travela_2022_12.py rename to bin/migrate/migrate_travela_2025_02.py index d5636d5e4..dd276ed86 100644 --- a/bin/migrate/migrate_travela_2022_12.py +++ b/bin/migrate/migrate_travela_2025_02.py @@ -11,20 +11,15 @@ import codecs -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] +sub_orgs = [ + 'dpm-vpm', + 'iga-aig', + 'miga-maig', + 'ghl-lgc', + 'mdi-mid', + 'ql-lq', + 'srp-rsp', + ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_travelq_2022_12.py b/bin/migrate/migrate_travelq_2025_02.py similarity index 64% rename from bin/migrate/migrate_travelq_2022_12.py rename to bin/migrate/migrate_travelq_2025_02.py index f9ae73ad1..251e8cb31 100644 --- a/bin/migrate/migrate_travelq_2022_12.py +++ b/bin/migrate/migrate_travelq_2025_02.py @@ -11,20 +11,15 @@ import codecs -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] +sub_orgs = [ + 'dpm-vpm', + 'iga-aig', + 'miga-maig', + 'ghl-lgc', + 'mdi-mid', + 'ql-lq', + 'srp-rsp', + ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } diff --git a/bin/migrate/migrate_travelq_nil_2022_12.py b/bin/migrate/migrate_travelq_nil_2025_02.py similarity index 76% rename from bin/migrate/migrate_travelq_nil_2022_12.py rename to bin/migrate/migrate_travelq_nil_2025_02.py index fadc91f58..a4c3c9caf 100644 --- a/bin/migrate/migrate_travelq_nil_2022_12.py +++ b/bin/migrate/migrate_travelq_nil_2025_02.py @@ -13,20 +13,15 @@ import codecs -sub_orgs = [ 'ghl-lgc', - 'iga-aig', - 'mdi-mid', - 'miga-maig', - 'dpm-vpm', - 'pkcc-pcprc', - 'ql-lq', - 'srp-rsp', - 'nsira-ossnr', - 'ocsec-bccst', - 'snsicp-scpssnr', - 'sirc-csars', - 'jfpc-cfp', - ] +sub_orgs = [ + 'dpm-vpm', + 'iga-aig', + 'miga-maig', + 'ghl-lgc', + 'mdi-mid', + 'ql-lq', + 'srp-rsp', + ] PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } From 8c53d3d42c90bd8ecd72281bcf1fbbc50ecc7eaf Mon Sep 17 00:00:00 2001 From: RabiaSajjad Date: Thu, 30 Jan 2025 14:30:29 -0500 Subject: [PATCH 11/15] fix linting error --- bin/migrate/migrate_briefingt_2025_02.py | 6 ++++-- bin/migrate/migrate_briefingt_nil_2025_12.py | 6 ++++-- bin/migrate/migrate_hospitalityq_2025_02.py | 6 ++++-- bin/migrate/migrate_hospitalityq_nil_2025_02.py | 6 ++++-- bin/migrate/migrate_qpnotes_2025_02.py | 6 ++++-- bin/migrate/migrate_travela_2025_02.py | 6 ++++-- bin/migrate/migrate_travelq_2025_02.py | 6 ++++-- bin/migrate/migrate_travelq_nil_2025_02.py | 6 ++++-- 8 files changed, 32 insertions(+), 16 deletions(-) diff --git a/bin/migrate/migrate_briefingt_2025_02.py b/bin/migrate/migrate_briefingt_2025_02.py index dfe89ab1e..ba1221a28 100644 --- a/bin/migrate/migrate_briefingt_2025_02.py +++ b/bin/migrate/migrate_briefingt_2025_02.py @@ -20,8 +20,10 @@ 'ql-lq', 'srp-rsp', ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } +PCO = { + 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', + } assert sys.stdin.read(3) == codecs.BOM_UTF8 diff --git a/bin/migrate/migrate_briefingt_nil_2025_12.py b/bin/migrate/migrate_briefingt_nil_2025_12.py index 940f1da7c..40f555da3 100644 --- a/bin/migrate/migrate_briefingt_nil_2025_12.py +++ b/bin/migrate/migrate_briefingt_nil_2025_12.py @@ -22,8 +22,10 @@ 'ql-lq', 'srp-rsp', ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } +PCO = { + 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', + } assert sys.stdin.read(3) == codecs.BOM_UTF8 diff --git a/bin/migrate/migrate_hospitalityq_2025_02.py b/bin/migrate/migrate_hospitalityq_2025_02.py index 31062c2c5..60f4bbef7 100644 --- a/bin/migrate/migrate_hospitalityq_2025_02.py +++ b/bin/migrate/migrate_hospitalityq_2025_02.py @@ -20,8 +20,10 @@ 'ql-lq', 'srp-rsp', ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } +PCO = { + 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', + } assert sys.stdin.read(3) == codecs.BOM_UTF8 diff --git a/bin/migrate/migrate_hospitalityq_nil_2025_02.py b/bin/migrate/migrate_hospitalityq_nil_2025_02.py index 1355fd765..574c9b553 100644 --- a/bin/migrate/migrate_hospitalityq_nil_2025_02.py +++ b/bin/migrate/migrate_hospitalityq_nil_2025_02.py @@ -22,8 +22,10 @@ 'ql-lq', 'srp-rsp', ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } +PCO = { + 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', + } assert sys.stdin.read(3) == codecs.BOM_UTF8 diff --git a/bin/migrate/migrate_qpnotes_2025_02.py b/bin/migrate/migrate_qpnotes_2025_02.py index 0dc45eb9b..d7feb27e6 100644 --- a/bin/migrate/migrate_qpnotes_2025_02.py +++ b/bin/migrate/migrate_qpnotes_2025_02.py @@ -20,8 +20,10 @@ 'ql-lq', 'srp-rsp', ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } +PCO = { + 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', + } assert sys.stdin.read(3) == codecs.BOM_UTF8 diff --git a/bin/migrate/migrate_travela_2025_02.py b/bin/migrate/migrate_travela_2025_02.py index dd276ed86..a4d49174b 100644 --- a/bin/migrate/migrate_travela_2025_02.py +++ b/bin/migrate/migrate_travela_2025_02.py @@ -20,8 +20,10 @@ 'ql-lq', 'srp-rsp', ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } +PCO = { + 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', + } assert sys.stdin.read(3) == codecs.BOM_UTF8 diff --git a/bin/migrate/migrate_travelq_2025_02.py b/bin/migrate/migrate_travelq_2025_02.py index 251e8cb31..a22f54992 100644 --- a/bin/migrate/migrate_travelq_2025_02.py +++ b/bin/migrate/migrate_travelq_2025_02.py @@ -20,8 +20,10 @@ 'ql-lq', 'srp-rsp', ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } +PCO = { + 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', + } assert sys.stdin.read(3) == codecs.BOM_UTF8 diff --git a/bin/migrate/migrate_travelq_nil_2025_02.py b/bin/migrate/migrate_travelq_nil_2025_02.py index a4c3c9caf..e75961d55 100644 --- a/bin/migrate/migrate_travelq_nil_2025_02.py +++ b/bin/migrate/migrate_travelq_nil_2025_02.py @@ -22,8 +22,10 @@ 'ql-lq', 'srp-rsp', ] -PCO = { 'owner_org': 'pco-bcp', - 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé' } +PCO = { + 'owner_org': 'pco-bcp', + 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', + } assert sys.stdin.read(3) == codecs.BOM_UTF8 From 556d633d8330691efcfbf78a437b8e484e610cc4 Mon Sep 17 00:00:00 2001 From: RabiaSajjad Date: Thu, 30 Jan 2025 14:31:37 -0500 Subject: [PATCH 12/15] add changelog for PR# 1342 --- changes/1342.changes | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/1342.changes diff --git a/changes/1342.changes b/changes/1342.changes new file mode 100644 index 000000000..0d4935abb --- /dev/null +++ b/changes/1342.changes @@ -0,0 +1 @@ +Merge PD records of PCO sub-organizations under PCO. From bc06c0d835f3896141a32c6d6f6f6b88e41fdf15 Mon Sep 17 00:00:00 2001 From: RabiaSajjad Date: Fri, 31 Jan 2025 17:20:05 -0500 Subject: [PATCH 13/15] improve migration scripts for python3 --- bin/migrate/migrate_briefingt_2025_02.py | 11 ++++++----- ...il_2025_12.py => migrate_briefingt_nil_2025_02.py} | 11 ++++++----- bin/migrate/migrate_hospitalityq_2025_02.py | 11 ++++++----- bin/migrate/migrate_hospitalityq_nil_2025_02.py | 11 ++++++----- bin/migrate/migrate_qpnotes_2025_02.py | 11 ++++++----- bin/migrate/migrate_travela_2025_02.py | 11 ++++++----- bin/migrate/migrate_travelq_2025_02.py | 11 ++++++----- bin/migrate/migrate_travelq_nil_2025_02.py | 11 ++++++----- 8 files changed, 48 insertions(+), 40 deletions(-) rename bin/migrate/{migrate_briefingt_nil_2025_12.py => migrate_briefingt_nil_2025_02.py} (82%) diff --git a/bin/migrate/migrate_briefingt_2025_02.py b/bin/migrate/migrate_briefingt_2025_02.py index ba1221a28..4b97f13ce 100644 --- a/bin/migrate/migrate_briefingt_2025_02.py +++ b/bin/migrate/migrate_briefingt_2025_02.py @@ -6,9 +6,9 @@ for PCO sub-organizations into PCO organization """ -import unicodecsv -import sys import codecs +import csv +import sys sub_orgs = [ @@ -20,15 +20,16 @@ 'ql-lq', 'srp-rsp', ] + PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.read(3) == codecs.BOM_UTF8 +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +in_csv = csv.DictReader(sys.stdin) +out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) out_csv.writeheader() for line in in_csv: diff --git a/bin/migrate/migrate_briefingt_nil_2025_12.py b/bin/migrate/migrate_briefingt_nil_2025_02.py similarity index 82% rename from bin/migrate/migrate_briefingt_nil_2025_12.py rename to bin/migrate/migrate_briefingt_nil_2025_02.py index 40f555da3..783c486f3 100644 --- a/bin/migrate/migrate_briefingt_nil_2025_12.py +++ b/bin/migrate/migrate_briefingt_nil_2025_02.py @@ -8,9 +8,9 @@ reporting_period will be ignored """ -import unicodecsv -import sys import codecs +import csv +import sys sub_orgs = [ @@ -22,15 +22,16 @@ 'ql-lq', 'srp-rsp', ] + PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.read(3) == codecs.BOM_UTF8 +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +in_csv = csv.DictReader(sys.stdin) +out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) out_csv.writeheader() data = [] diff --git a/bin/migrate/migrate_hospitalityq_2025_02.py b/bin/migrate/migrate_hospitalityq_2025_02.py index 60f4bbef7..287d4db6f 100644 --- a/bin/migrate/migrate_hospitalityq_2025_02.py +++ b/bin/migrate/migrate_hospitalityq_2025_02.py @@ -6,9 +6,9 @@ for PCO sub-organizations into PCO organization """ -import unicodecsv -import sys import codecs +import csv +import sys sub_orgs = [ @@ -20,15 +20,16 @@ 'ql-lq', 'srp-rsp', ] + PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.read(3) == codecs.BOM_UTF8 +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +in_csv = csv.DictReader(sys.stdin) +out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) out_csv.writeheader() for line in in_csv: diff --git a/bin/migrate/migrate_hospitalityq_nil_2025_02.py b/bin/migrate/migrate_hospitalityq_nil_2025_02.py index 574c9b553..cde0c26c2 100644 --- a/bin/migrate/migrate_hospitalityq_nil_2025_02.py +++ b/bin/migrate/migrate_hospitalityq_nil_2025_02.py @@ -8,9 +8,9 @@ will be ignored """ -import unicodecsv -import sys import codecs +import csv +import sys sub_orgs = [ @@ -22,15 +22,16 @@ 'ql-lq', 'srp-rsp', ] + PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.read(3) == codecs.BOM_UTF8 +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +in_csv = csv.DictReader(sys.stdin) +out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) out_csv.writeheader() data = [] diff --git a/bin/migrate/migrate_qpnotes_2025_02.py b/bin/migrate/migrate_qpnotes_2025_02.py index d7feb27e6..358114537 100644 --- a/bin/migrate/migrate_qpnotes_2025_02.py +++ b/bin/migrate/migrate_qpnotes_2025_02.py @@ -6,9 +6,9 @@ for PCO sub-organizations into PCO organization """ -import unicodecsv -import sys import codecs +import csv +import sys sub_orgs = [ @@ -20,15 +20,16 @@ 'ql-lq', 'srp-rsp', ] + PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.read(3) == codecs.BOM_UTF8 +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +in_csv = csv.DictReader(sys.stdin) +out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) out_csv.writeheader() for line in in_csv: diff --git a/bin/migrate/migrate_travela_2025_02.py b/bin/migrate/migrate_travela_2025_02.py index a4d49174b..177eb9518 100644 --- a/bin/migrate/migrate_travela_2025_02.py +++ b/bin/migrate/migrate_travela_2025_02.py @@ -6,9 +6,9 @@ for PCO sub-organizations into PCO organization """ -import unicodecsv -import sys import codecs +import csv +import sys sub_orgs = [ @@ -20,15 +20,16 @@ 'ql-lq', 'srp-rsp', ] + PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.read(3) == codecs.BOM_UTF8 +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +in_csv = csv.DictReader(sys.stdin) +out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) out_csv.writeheader() for line in in_csv: diff --git a/bin/migrate/migrate_travelq_2025_02.py b/bin/migrate/migrate_travelq_2025_02.py index a22f54992..23d245090 100644 --- a/bin/migrate/migrate_travelq_2025_02.py +++ b/bin/migrate/migrate_travelq_2025_02.py @@ -6,9 +6,9 @@ for PCO sub-organizations into PCO organization """ -import unicodecsv -import sys import codecs +import csv +import sys sub_orgs = [ @@ -20,15 +20,16 @@ 'ql-lq', 'srp-rsp', ] + PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.read(3) == codecs.BOM_UTF8 +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +in_csv = csv.DictReader(sys.stdin) +out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) out_csv.writeheader() for line in in_csv: diff --git a/bin/migrate/migrate_travelq_nil_2025_02.py b/bin/migrate/migrate_travelq_nil_2025_02.py index e75961d55..e5ca7ed24 100644 --- a/bin/migrate/migrate_travelq_nil_2025_02.py +++ b/bin/migrate/migrate_travelq_nil_2025_02.py @@ -8,9 +8,9 @@ will be ignored """ -import unicodecsv -import sys import codecs +import csv +import sys sub_orgs = [ @@ -22,15 +22,16 @@ 'ql-lq', 'srp-rsp', ] + PCO = { 'owner_org': 'pco-bcp', 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.read(3) == codecs.BOM_UTF8 +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." -in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') -out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames, encoding='utf-8') +in_csv = csv.DictReader(sys.stdin) +out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) out_csv.writeheader() data = [] From 455ea5196a46843b3a9b8217ab177eded2d713cd Mon Sep 17 00:00:00 2001 From: RabiaSajjad Date: Fri, 31 Jan 2025 17:25:29 -0500 Subject: [PATCH 14/15] remove linting error from pco migration scripts --- bin/migrate/migrate_briefingt_2025_02.py | 2 +- bin/migrate/migrate_briefingt_nil_2025_02.py | 2 +- bin/migrate/migrate_hospitalityq_2025_02.py | 2 +- bin/migrate/migrate_hospitalityq_nil_2025_02.py | 2 +- bin/migrate/migrate_qpnotes_2025_02.py | 2 +- bin/migrate/migrate_travela_2025_02.py | 2 +- bin/migrate/migrate_travelq_2025_02.py | 2 +- bin/migrate/migrate_travelq_nil_2025_02.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/migrate/migrate_briefingt_2025_02.py b/bin/migrate/migrate_briefingt_2025_02.py index 4b97f13ce..a46a03bfc 100644 --- a/bin/migrate/migrate_briefingt_2025_02.py +++ b/bin/migrate/migrate_briefingt_2025_02.py @@ -26,7 +26,7 @@ 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8 in_csv = csv.DictReader(sys.stdin) out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) diff --git a/bin/migrate/migrate_briefingt_nil_2025_02.py b/bin/migrate/migrate_briefingt_nil_2025_02.py index 783c486f3..6251fdb9a 100644 --- a/bin/migrate/migrate_briefingt_nil_2025_02.py +++ b/bin/migrate/migrate_briefingt_nil_2025_02.py @@ -28,7 +28,7 @@ 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8 in_csv = csv.DictReader(sys.stdin) out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) diff --git a/bin/migrate/migrate_hospitalityq_2025_02.py b/bin/migrate/migrate_hospitalityq_2025_02.py index 287d4db6f..eed081af9 100644 --- a/bin/migrate/migrate_hospitalityq_2025_02.py +++ b/bin/migrate/migrate_hospitalityq_2025_02.py @@ -26,7 +26,7 @@ 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8 in_csv = csv.DictReader(sys.stdin) out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) diff --git a/bin/migrate/migrate_hospitalityq_nil_2025_02.py b/bin/migrate/migrate_hospitalityq_nil_2025_02.py index cde0c26c2..07475b0d4 100644 --- a/bin/migrate/migrate_hospitalityq_nil_2025_02.py +++ b/bin/migrate/migrate_hospitalityq_nil_2025_02.py @@ -28,7 +28,7 @@ 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8 in_csv = csv.DictReader(sys.stdin) out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) diff --git a/bin/migrate/migrate_qpnotes_2025_02.py b/bin/migrate/migrate_qpnotes_2025_02.py index 358114537..37ededef0 100644 --- a/bin/migrate/migrate_qpnotes_2025_02.py +++ b/bin/migrate/migrate_qpnotes_2025_02.py @@ -26,7 +26,7 @@ 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8 in_csv = csv.DictReader(sys.stdin) out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) diff --git a/bin/migrate/migrate_travela_2025_02.py b/bin/migrate/migrate_travela_2025_02.py index 177eb9518..75914b2c6 100644 --- a/bin/migrate/migrate_travela_2025_02.py +++ b/bin/migrate/migrate_travela_2025_02.py @@ -26,7 +26,7 @@ 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8 in_csv = csv.DictReader(sys.stdin) out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) diff --git a/bin/migrate/migrate_travelq_2025_02.py b/bin/migrate/migrate_travelq_2025_02.py index 23d245090..0b2fc03a9 100644 --- a/bin/migrate/migrate_travelq_2025_02.py +++ b/bin/migrate/migrate_travelq_2025_02.py @@ -26,7 +26,7 @@ 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8 in_csv = csv.DictReader(sys.stdin) out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) diff --git a/bin/migrate/migrate_travelq_nil_2025_02.py b/bin/migrate/migrate_travelq_nil_2025_02.py index e5ca7ed24..3487e9f70 100644 --- a/bin/migrate/migrate_travelq_nil_2025_02.py +++ b/bin/migrate/migrate_travelq_nil_2025_02.py @@ -28,7 +28,7 @@ 'owner_org_title': 'Privy Council Office | Bureau du Conseil privé', } -assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8, "Input file does not contain the UTF-8 BOM." +assert sys.stdin.buffer.read(3) == codecs.BOM_UTF8 in_csv = csv.DictReader(sys.stdin) out_csv = csv.DictWriter(sys.stdout, fieldnames=in_csv.fieldnames) From f285bdedc1680a3059f5fc16f6e0a9e1fe2ca79c Mon Sep 17 00:00:00 2001 From: RabiaSajjad Date: Fri, 31 Jan 2025 17:28:01 -0500 Subject: [PATCH 15/15] remove more linting error from pco migration scripts --- bin/migrate/migrate_briefingt_nil_2025_02.py | 2 ++ bin/migrate/migrate_hospitalityq_nil_2025_02.py | 2 ++ bin/migrate/migrate_travelq_nil_2025_02.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/bin/migrate/migrate_briefingt_nil_2025_02.py b/bin/migrate/migrate_briefingt_nil_2025_02.py index 6251fdb9a..cb6a40deb 100644 --- a/bin/migrate/migrate_briefingt_nil_2025_02.py +++ b/bin/migrate/migrate_briefingt_nil_2025_02.py @@ -36,6 +36,7 @@ data = [] + def report_exists(reporting_period, owner_org): for row in data: if row['reporting_period'] == reporting_period and \ @@ -43,6 +44,7 @@ def report_exists(reporting_period, owner_org): return True return False + for line in in_csv: if line['owner_org'] in sub_orgs: if not report_exists(line['reporting_period'], PCO['owner_org']): diff --git a/bin/migrate/migrate_hospitalityq_nil_2025_02.py b/bin/migrate/migrate_hospitalityq_nil_2025_02.py index 07475b0d4..7c691b221 100644 --- a/bin/migrate/migrate_hospitalityq_nil_2025_02.py +++ b/bin/migrate/migrate_hospitalityq_nil_2025_02.py @@ -36,6 +36,7 @@ data = [] + def report_exists(year, month, owner_org): for row in data: if row['year'] == year and \ @@ -44,6 +45,7 @@ def report_exists(year, month, owner_org): return True return False + for line in in_csv: if line['owner_org'] in sub_orgs: if not report_exists(line['year'], line['month'], PCO['owner_org']): diff --git a/bin/migrate/migrate_travelq_nil_2025_02.py b/bin/migrate/migrate_travelq_nil_2025_02.py index 3487e9f70..4045c88be 100644 --- a/bin/migrate/migrate_travelq_nil_2025_02.py +++ b/bin/migrate/migrate_travelq_nil_2025_02.py @@ -36,6 +36,7 @@ data = [] + def report_exists(year, month, owner_org): for row in data: if row['year'] == year and \ @@ -44,6 +45,7 @@ def report_exists(year, month, owner_org): return True return False + for line in in_csv: if line['owner_org'] in sub_orgs: if not report_exists(line['year'], line['month'], PCO['owner_org']):