From 8825ff0298366f1fc43767c30f962185d1d0cff9 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Thu, 4 Jan 2024 16:55:41 +0200 Subject: [PATCH 01/11] Update old mysql packages --- requirements.api.txt | 2 +- requirements.dev.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.api.txt b/requirements.api.txt index c1ac3c2f1..09adc792e 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -4,7 +4,7 @@ Flask==2.2.5 Flask-Limiter==3.3.0 jinja2==3.0.3 more_itertools==8.4.0 -mysqlclient==2.1.1 +mysqlclient==2.2.1 orjson==3.4.7 pandas==1.2.3 python-dotenv==0.15.0 diff --git a/requirements.dev.txt b/requirements.dev.txt index c071f208d..c1e3358c2 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -10,10 +10,10 @@ invoke>=1.4.1 lxml==4.9.1 matplotlib==3.6.2 mypy>=0.790 -mysql-connector==2.2.9 +mysql-connector-python==8.2.0 numpy==1.22.4 pycountry==22.3.5 -pymysql==1.0.2 +pymysql==1.1.0 pytest==7.2.0 pytest-check==1.3.0 sas7bdat==2.2.3 From 9888b5dcc66c30232f9fd961c23e0e6e9f2a81ea Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 18:53:21 +0200 Subject: [PATCH 02/11] Update --- .../acquisition/covidcast/test_covidcast_meta_caching.py | 8 ++++---- integrations/acquisition/covidcast/test_csv_uploading.py | 6 +++--- integrations/server/test_covidcast.py | 4 ++-- integrations/server/test_covidcast_meta.py | 4 ++-- integrations/server/test_fluview.py | 4 ++-- integrations/server/test_fluview_meta.py | 4 ++-- requirements.api.txt | 2 +- requirements.dev.txt | 2 -- src/acquisition/cdcp/cdc_dropbox_receiver.py | 4 ++-- src/acquisition/cdcp/cdc_extract.py | 4 ++-- src/acquisition/cdcp/cdc_upload.py | 4 ++-- src/acquisition/covid_hosp/common/database.py | 4 ++-- src/acquisition/covidcast/database.py | 4 ++-- src/acquisition/ecdc/ecdc_db_update.py | 6 +++--- src/acquisition/flusurv/flusurv_update.py | 4 ++-- src/acquisition/fluview/fluview_notify.py | 4 ++-- src/acquisition/fluview/fluview_update.py | 8 ++++---- src/acquisition/fluview/impute_missing_values.py | 4 ++-- src/acquisition/ght/ght_update.py | 4 ++-- src/acquisition/kcdc/kcdc_update.py | 6 +++--- src/acquisition/nidss/taiwan_update.py | 4 ++-- src/acquisition/paho/paho_db_update.py | 6 +++--- src/acquisition/quidel/quidel_update.py | 4 ++-- src/acquisition/twtr/twitter_update.py | 4 ++-- src/acquisition/wiki/wiki_extract.py | 4 ++-- src/acquisition/wiki/wiki_update.py | 4 ++-- src/common/integration_test_base_class.py | 4 ++-- src/maintenance/remove_outdated_keys.py | 4 ++-- src/maintenance/signal_dash_data_generator.py | 8 ++++---- src/maintenance/update_last_usage.py | 4 ++-- 30 files changed, 67 insertions(+), 69 deletions(-) diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 30b6bbcf2..de06225e0 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -5,7 +5,7 @@ import unittest # third party -import mysql.connector +import MySQLdb import requests # first party @@ -32,11 +32,11 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', - database='covid') + db='covid') cur = cnx.cursor() # clear all tables @@ -58,7 +58,7 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') - epidata_cnx = mysql.connector.connect( + epidata_cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index dab35f414..968484762 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -7,7 +7,7 @@ import argparse # third party -import mysql.connector +import MySQLdb import pandas as pd import numpy as np @@ -28,7 +28,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', @@ -55,7 +55,7 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') - epidata_cnx = mysql.connector.connect( + epidata_cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 8b7fc6f52..01527fa90 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -5,7 +5,7 @@ import unittest # third party -import mysql.connector +import MySQLdb # first party from delphi_utils import Nans @@ -369,7 +369,7 @@ def test_unique_key_constraint(self): row = self._insert_placeholder_set_one() # fail to insert different placeholder data under the same key - with self.assertRaises(mysql.connector.errors.IntegrityError): + with self.assertRaises(MySQLdb._exceptions.IntegrityError): self._insert_placeholder_set_one() # succeed to insert different placeholder data under a different time_type diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 857422a41..7fba26cbb 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -4,7 +4,7 @@ import unittest # third party -import mysql.connector +import MySQLdb import requests #first party @@ -53,7 +53,7 @@ def localSetUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index 48d9585fd..b6a7719c5 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -4,7 +4,7 @@ import unittest # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -25,7 +25,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `fluview` table - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 6f81c1859..2e23a994a 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -4,7 +4,7 @@ import unittest # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -25,7 +25,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `fluview` table - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/requirements.api.txt b/requirements.api.txt index fe66b0468..c0f51d837 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -13,7 +13,7 @@ redis==3.5.3 requests==2.31.0 scipy==1.10.0 sentry-sdk[flask] -SQLAlchemy==1.4.40 +SQLAlchemy==1.4.51 structlog==22.1.0 tenacity==7.0.0 typing-extensions diff --git a/requirements.dev.txt b/requirements.dev.txt index c1e3358c2..3f0d4f2f2 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -10,10 +10,8 @@ invoke>=1.4.1 lxml==4.9.1 matplotlib==3.6.2 mypy>=0.790 -mysql-connector-python==8.2.0 numpy==1.22.4 pycountry==22.3.5 -pymysql==1.1.0 pytest==7.2.0 pytest-check==1.3.0 sas7bdat==2.2.3 diff --git a/src/acquisition/cdcp/cdc_dropbox_receiver.py b/src/acquisition/cdcp/cdc_dropbox_receiver.py index 4fa20368e..9713aa384 100644 --- a/src/acquisition/cdcp/cdc_dropbox_receiver.py +++ b/src/acquisition/cdcp/cdc_dropbox_receiver.py @@ -22,7 +22,7 @@ # third party import dropbox -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -47,7 +47,7 @@ def trigger_further_processing(): # connect u, p = secrets.db.auto - cnx = mysql.connector.connect(user=u, password=p, database="automation") + cnx = MySQLdb.connect(user=u, password=p, database="automation") cur = cnx.cursor() # add step "Process CDCP Data" to queue diff --git a/src/acquisition/cdcp/cdc_extract.py b/src/acquisition/cdcp/cdc_extract.py index 0d38e0bcc..13237c2da 100644 --- a/src/acquisition/cdcp/cdc_extract.py +++ b/src/acquisition/cdcp/cdc_extract.py @@ -66,7 +66,7 @@ import sys # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -154,7 +154,7 @@ def extract(first_week=None, last_week=None, test_mode=False): # connect u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # weeks to update diff --git a/src/acquisition/cdcp/cdc_upload.py b/src/acquisition/cdcp/cdc_upload.py index 0e191267b..e154f2e62 100644 --- a/src/acquisition/cdcp/cdc_upload.py +++ b/src/acquisition/cdcp/cdc_upload.py @@ -80,7 +80,7 @@ from zipfile import ZipFile # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -165,7 +165,7 @@ def upload(test_mode): # connect u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # insert (or update) table `cdc` diff --git a/src/acquisition/covid_hosp/common/database.py b/src/acquisition/covid_hosp/common/database.py index efbdb6c45..1c79c37c5 100644 --- a/src/acquisition/covid_hosp/common/database.py +++ b/src/acquisition/covid_hosp/common/database.py @@ -6,7 +6,7 @@ import math # third party -import mysql.connector +import MySQLdb import pandas as pd # first party @@ -60,7 +60,7 @@ def logger(database_class): @classmethod @contextmanager - def connect(database_class, mysql_connector_impl=mysql.connector): + def connect(database_class, mysql_connector_impl=MySQLdb): """Connect to a database and provide the connection as a context manager. As long as the context manager exits normally, the connection's transaction diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 871061b81..0a8e359f5 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -10,7 +10,7 @@ # third party import json -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -38,7 +38,7 @@ class Database: # TODO: also consider that for composite key tuples, like short_comp_key and long_comp_key as used in delete_batch() - def connect(self, connector_impl=mysql.connector): + def connect(self, connector_impl=MySQLdb): """Establish a connection to the database.""" u, p = secrets.db.epi diff --git a/src/acquisition/ecdc/ecdc_db_update.py b/src/acquisition/ecdc/ecdc_db_update.py index 84423c376..70f4c3cee 100644 --- a/src/acquisition/ecdc/ecdc_db_update.py +++ b/src/acquisition/ecdc/ecdc_db_update.py @@ -37,7 +37,7 @@ import tempfile # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -48,7 +48,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -98,7 +98,7 @@ def update_from_file(issue, date, dir, test_mode=False): # Read ECDC data from CSVs and insert into (or update) the database. # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx, "ecdc_ili") print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/flusurv/flusurv_update.py b/src/acquisition/flusurv/flusurv_update.py index 1aa8e9885..17490fcb1 100644 --- a/src/acquisition/flusurv/flusurv_update.py +++ b/src/acquisition/flusurv/flusurv_update.py @@ -72,7 +72,7 @@ import argparse # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.acquisition.flusurv import flusurv @@ -105,7 +105,7 @@ def update(issue, location_name, test_mode=False): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(host=secrets.db.host, user=u, password=p, database="epidata") + cnx = MySQLdb.connect(host=secrets.db.host, user=u, password=p, database="epidata") cur = cnx.cursor() rows1 = get_rows(cur) print(f"rows before: {int(rows1)}") diff --git a/src/acquisition/fluview/fluview_notify.py b/src/acquisition/fluview/fluview_notify.py index 3ed1a243f..e597d2098 100644 --- a/src/acquisition/fluview/fluview_notify.py +++ b/src/acquisition/fluview/fluview_notify.py @@ -25,7 +25,7 @@ import argparse # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -46,7 +46,7 @@ # connect u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) cur = cnx.cursor() # get the last known issue from the automation table `variables` diff --git a/src/acquisition/fluview/fluview_update.py b/src/acquisition/fluview/fluview_update.py index defd01dad..f292c281d 100644 --- a/src/acquisition/fluview/fluview_update.py +++ b/src/acquisition/fluview/fluview_update.py @@ -121,7 +121,7 @@ import zipfile # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -311,7 +311,7 @@ def update_from_file_clinical(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx, CL_TABLE) print(f"rows before: {int(rows1)}") insert = cnx.cursor() @@ -376,7 +376,7 @@ def update_from_file_public(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx, PHL_TABLE) print(f"rows before: {int(rows1)}") insert = cnx.cursor() @@ -445,7 +445,7 @@ def update_from_file(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx) print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/fluview/impute_missing_values.py b/src/acquisition/fluview/impute_missing_values.py index c795d9cce..fc0d8d357 100644 --- a/src/acquisition/fluview/impute_missing_values.py +++ b/src/acquisition/fluview/impute_missing_values.py @@ -49,7 +49,7 @@ import argparse # third party -import mysql.connector +import MySQLdb import numpy as np # first party @@ -135,7 +135,7 @@ class Sql: def connect(self): """Connect to the database.""" u, p = secrets.db.epi - self.cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + self.cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) self.cur = self.cnx.cursor() def close(self, commit): diff --git a/src/acquisition/ght/ght_update.py b/src/acquisition/ght/ght_update.py index 9e8d48d1d..a58283d76 100644 --- a/src/acquisition/ght/ght_update.py +++ b/src/acquisition/ght/ght_update.py @@ -70,7 +70,7 @@ import time # third party -import mysql.connector +import MySQLdb from apiclient.discovery import build # first party @@ -251,7 +251,7 @@ def update(locations, terms, first=None, last=None, countries=["US"]): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/kcdc/kcdc_update.py b/src/acquisition/kcdc/kcdc_update.py index 713b21f00..1fae58c95 100644 --- a/src/acquisition/kcdc/kcdc_update.py +++ b/src/acquisition/kcdc/kcdc_update.py @@ -35,7 +35,7 @@ import requests # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -45,7 +45,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -124,7 +124,7 @@ def get_kcdc_data(): def update_from_data(ews, ilis, date, issue, test_mode=False): u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx) print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/nidss/taiwan_update.py b/src/acquisition/nidss/taiwan_update.py index 30d458481..2de4ab8b8 100644 --- a/src/acquisition/nidss/taiwan_update.py +++ b/src/acquisition/nidss/taiwan_update.py @@ -77,7 +77,7 @@ import argparse # third party -import mysql.connector +import MySQLdb # first party from .taiwan_nidss import NIDSS @@ -105,7 +105,7 @@ def update(test_mode=False): # Database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx) print(f"rows before (flu): {int(rows1[0])}") print(f"rows before (dengue): {int(rows1[1])}") diff --git a/src/acquisition/paho/paho_db_update.py b/src/acquisition/paho/paho_db_update.py index b351d3ff2..0dfd08e40 100644 --- a/src/acquisition/paho/paho_db_update.py +++ b/src/acquisition/paho/paho_db_update.py @@ -54,7 +54,7 @@ import tempfile # third party -import mysql.connector +import MySQLdb import pycountry # first party @@ -66,7 +66,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -169,7 +169,7 @@ def update_from_file(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx, "paho_dengue") print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/quidel/quidel_update.py b/src/acquisition/quidel/quidel_update.py index 563cea898..a1ced5652 100644 --- a/src/acquisition/quidel/quidel_update.py +++ b/src/acquisition/quidel/quidel_update.py @@ -39,7 +39,7 @@ import argparse # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.acquisition.quidel import quidel @@ -64,7 +64,7 @@ def update(locations, first=None, last=None, force_update=False, load_email=True qd_ts = quidel.measurement_to_ts(qd_measurements, 7, startweek=first, endweek=last) # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/twtr/twitter_update.py b/src/acquisition/twtr/twitter_update.py index 80a023f19..c89681af2 100644 --- a/src/acquisition/twtr/twitter_update.py +++ b/src/acquisition/twtr/twitter_update.py @@ -52,7 +52,7 @@ """ # third party -import mysql.connector +import MySQLdb # first party from .healthtweets import HealthTweets @@ -62,7 +62,7 @@ def run(): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/wiki/wiki_extract.py b/src/acquisition/wiki/wiki_extract.py index 718a64c20..5f886f2e5 100644 --- a/src/acquisition/wiki/wiki_extract.py +++ b/src/acquisition/wiki/wiki_extract.py @@ -28,7 +28,7 @@ import json # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -67,7 +67,7 @@ def get_timestamp(name): def run(job_limit=100): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # # Some preparation for utf-8, and it is a temporary trick solution. The real solution should change those char set and collation encoding to utf8 permanently diff --git a/src/acquisition/wiki/wiki_update.py b/src/acquisition/wiki/wiki_update.py index a9f240629..667ac9b46 100644 --- a/src/acquisition/wiki/wiki_update.py +++ b/src/acquisition/wiki/wiki_update.py @@ -24,7 +24,7 @@ from datetime import datetime, timedelta # third party -import mysql.connector +import MySQLdb import requests # first party @@ -90,7 +90,7 @@ def get_manifest(year, month, optional=False): def run(): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # get the most recent job in wiki_raw diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 47c9f68e4..67fda472d 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -2,7 +2,7 @@ import unittest # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -33,7 +33,7 @@ def setUp(self) -> None: """Perform per-test setup.""" # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cnx = MySQLdb.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") cur = cnx.cursor() cur.execute("DELETE FROM `api_user`") diff --git a/src/maintenance/remove_outdated_keys.py b/src/maintenance/remove_outdated_keys.py index 56fea0cf9..263565cae 100644 --- a/src/maintenance/remove_outdated_keys.py +++ b/src/maintenance/remove_outdated_keys.py @@ -2,7 +2,7 @@ from smtplib import SMTP import delphi.operations.secrets as secrets -import mysql.connector +import MySQLdb from delphi.epidata.server._config import API_KEY_REGISTRATION_FORM_LINK_LOCAL ApiUserRecord = namedtuple("APIUserRecord", ("api_key", "email", "date_diff")) @@ -51,7 +51,7 @@ def send_notification(to_addr, alert=True): def main(): u, p = secrets.db.epi - cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) + cnx = MySQLdb.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() outdated_keys_list = get_old_keys(cur) for item in outdated_keys_list: diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index b7f1048f5..75ec2fe9b 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -5,7 +5,7 @@ import sys import time import datetime -import mysql.connector +import MySQLdb import pandas as pd from dataclasses import dataclass @@ -62,7 +62,7 @@ class Database: STATUS_TABLE_NAME = 'dashboard_signal_status' COVERAGE_TABLE_NAME = 'dashboard_signal_coverage' - def __init__(self, connector_impl=mysql.connector): + def __init__(self, connector_impl=MySQLdb): """Establish a connection to the database.""" u, p = secrets.db.epi @@ -264,13 +264,13 @@ def main(args): try: database.write_status(signal_status_list) logger.info("Wrote status.", rowcount=database.rowcount()) - except mysql.connector.Error as exception: + except MySQLdb.MySQLError as exception: logger.exception(exception) try: database.write_coverage(coverage_list) logger.info("Wrote coverage.", rowcount=database.rowcount()) - except mysql.connector.Error as exception: + except MySQLdb.MySQLError as exception: logger.exception(exception) logger.info( diff --git a/src/maintenance/update_last_usage.py b/src/maintenance/update_last_usage.py index 85596778a..dd9a52a58 100644 --- a/src/maintenance/update_last_usage.py +++ b/src/maintenance/update_last_usage.py @@ -2,7 +2,7 @@ from datetime import datetime as dtime import delphi.operations.secrets as secrets -import mysql.connector +import MySQLdb import redis REDIS_HOST = os.environ.get("REDIS_HOST", "delphi_redis") @@ -13,7 +13,7 @@ def main(): redis_cli = redis.Redis(host=REDIS_HOST, password=REDIS_PASSWORD, decode_responses=True) u, p = secrets.db.epi - cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) + cnx = MySQLdb.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() redis_keys = redis_cli.keys(pattern=LAST_USED_KEY_PATTERN) From 6b821c72f2a6800fac306613ed9f2fb41ce210f3 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 19:10:58 +0200 Subject: [PATCH 03/11] Revert "Update" This reverts commit 9888b5dcc66c30232f9fd961c23e0e6e9f2a81ea. --- .../acquisition/covidcast/test_covidcast_meta_caching.py | 8 ++++---- integrations/acquisition/covidcast/test_csv_uploading.py | 6 +++--- integrations/server/test_covidcast.py | 4 ++-- integrations/server/test_covidcast_meta.py | 4 ++-- integrations/server/test_fluview.py | 4 ++-- integrations/server/test_fluview_meta.py | 4 ++-- requirements.api.txt | 2 +- requirements.dev.txt | 2 ++ src/acquisition/cdcp/cdc_dropbox_receiver.py | 4 ++-- src/acquisition/cdcp/cdc_extract.py | 4 ++-- src/acquisition/cdcp/cdc_upload.py | 4 ++-- src/acquisition/covid_hosp/common/database.py | 4 ++-- src/acquisition/covidcast/database.py | 4 ++-- src/acquisition/ecdc/ecdc_db_update.py | 6 +++--- src/acquisition/flusurv/flusurv_update.py | 4 ++-- src/acquisition/fluview/fluview_notify.py | 4 ++-- src/acquisition/fluview/fluview_update.py | 8 ++++---- src/acquisition/fluview/impute_missing_values.py | 4 ++-- src/acquisition/ght/ght_update.py | 4 ++-- src/acquisition/kcdc/kcdc_update.py | 6 +++--- src/acquisition/nidss/taiwan_update.py | 4 ++-- src/acquisition/paho/paho_db_update.py | 6 +++--- src/acquisition/quidel/quidel_update.py | 4 ++-- src/acquisition/twtr/twitter_update.py | 4 ++-- src/acquisition/wiki/wiki_extract.py | 4 ++-- src/acquisition/wiki/wiki_update.py | 4 ++-- src/common/integration_test_base_class.py | 4 ++-- src/maintenance/remove_outdated_keys.py | 4 ++-- src/maintenance/signal_dash_data_generator.py | 8 ++++---- src/maintenance/update_last_usage.py | 4 ++-- 30 files changed, 69 insertions(+), 67 deletions(-) diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index de06225e0..30b6bbcf2 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -5,7 +5,7 @@ import unittest # third party -import MySQLdb +import mysql.connector import requests # first party @@ -32,11 +32,11 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database - cnx = MySQLdb.connect( + cnx = mysql.connector.connect( user='user', password='pass', host='delphi_database_epidata', - db='covid') + database='covid') cur = cnx.cursor() # clear all tables @@ -58,7 +58,7 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') - epidata_cnx = MySQLdb.connect( + epidata_cnx = mysql.connector.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index 968484762..dab35f414 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -7,7 +7,7 @@ import argparse # third party -import MySQLdb +import mysql.connector import pandas as pd import numpy as np @@ -28,7 +28,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `covidcast` table - cnx = MySQLdb.connect( + cnx = mysql.connector.connect( user='user', password='pass', host='delphi_database_epidata', @@ -55,7 +55,7 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') - epidata_cnx = MySQLdb.connect( + epidata_cnx = mysql.connector.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 01527fa90..8b7fc6f52 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -5,7 +5,7 @@ import unittest # third party -import MySQLdb +import mysql.connector # first party from delphi_utils import Nans @@ -369,7 +369,7 @@ def test_unique_key_constraint(self): row = self._insert_placeholder_set_one() # fail to insert different placeholder data under the same key - with self.assertRaises(MySQLdb._exceptions.IntegrityError): + with self.assertRaises(mysql.connector.errors.IntegrityError): self._insert_placeholder_set_one() # succeed to insert different placeholder data under a different time_type diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 7fba26cbb..857422a41 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -4,7 +4,7 @@ import unittest # third party -import MySQLdb +import mysql.connector import requests #first party @@ -53,7 +53,7 @@ def localSetUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `covidcast` table - cnx = MySQLdb.connect( + cnx = mysql.connector.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index b6a7719c5..48d9585fd 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -4,7 +4,7 @@ import unittest # third party -import MySQLdb +import mysql.connector # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -25,7 +25,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `fluview` table - cnx = MySQLdb.connect( + cnx = mysql.connector.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 2e23a994a..6f81c1859 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -4,7 +4,7 @@ import unittest # third party -import MySQLdb +import mysql.connector # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -25,7 +25,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `fluview` table - cnx = MySQLdb.connect( + cnx = mysql.connector.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/requirements.api.txt b/requirements.api.txt index c0f51d837..fe66b0468 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -13,7 +13,7 @@ redis==3.5.3 requests==2.31.0 scipy==1.10.0 sentry-sdk[flask] -SQLAlchemy==1.4.51 +SQLAlchemy==1.4.40 structlog==22.1.0 tenacity==7.0.0 typing-extensions diff --git a/requirements.dev.txt b/requirements.dev.txt index 3f0d4f2f2..c1e3358c2 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -10,8 +10,10 @@ invoke>=1.4.1 lxml==4.9.1 matplotlib==3.6.2 mypy>=0.790 +mysql-connector-python==8.2.0 numpy==1.22.4 pycountry==22.3.5 +pymysql==1.1.0 pytest==7.2.0 pytest-check==1.3.0 sas7bdat==2.2.3 diff --git a/src/acquisition/cdcp/cdc_dropbox_receiver.py b/src/acquisition/cdcp/cdc_dropbox_receiver.py index 9713aa384..4fa20368e 100644 --- a/src/acquisition/cdcp/cdc_dropbox_receiver.py +++ b/src/acquisition/cdcp/cdc_dropbox_receiver.py @@ -22,7 +22,7 @@ # third party import dropbox -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -47,7 +47,7 @@ def trigger_further_processing(): # connect u, p = secrets.db.auto - cnx = MySQLdb.connect(user=u, password=p, database="automation") + cnx = mysql.connector.connect(user=u, password=p, database="automation") cur = cnx.cursor() # add step "Process CDCP Data" to queue diff --git a/src/acquisition/cdcp/cdc_extract.py b/src/acquisition/cdcp/cdc_extract.py index 13237c2da..0d38e0bcc 100644 --- a/src/acquisition/cdcp/cdc_extract.py +++ b/src/acquisition/cdcp/cdc_extract.py @@ -66,7 +66,7 @@ import sys # third party -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -154,7 +154,7 @@ def extract(first_week=None, last_week=None, test_mode=False): # connect u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # weeks to update diff --git a/src/acquisition/cdcp/cdc_upload.py b/src/acquisition/cdcp/cdc_upload.py index e154f2e62..0e191267b 100644 --- a/src/acquisition/cdcp/cdc_upload.py +++ b/src/acquisition/cdcp/cdc_upload.py @@ -80,7 +80,7 @@ from zipfile import ZipFile # third party -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -165,7 +165,7 @@ def upload(test_mode): # connect u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # insert (or update) table `cdc` diff --git a/src/acquisition/covid_hosp/common/database.py b/src/acquisition/covid_hosp/common/database.py index 1c79c37c5..efbdb6c45 100644 --- a/src/acquisition/covid_hosp/common/database.py +++ b/src/acquisition/covid_hosp/common/database.py @@ -6,7 +6,7 @@ import math # third party -import MySQLdb +import mysql.connector import pandas as pd # first party @@ -60,7 +60,7 @@ def logger(database_class): @classmethod @contextmanager - def connect(database_class, mysql_connector_impl=MySQLdb): + def connect(database_class, mysql_connector_impl=mysql.connector): """Connect to a database and provide the connection as a context manager. As long as the context manager exits normally, the connection's transaction diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 0a8e359f5..871061b81 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -10,7 +10,7 @@ # third party import json -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -38,7 +38,7 @@ class Database: # TODO: also consider that for composite key tuples, like short_comp_key and long_comp_key as used in delete_batch() - def connect(self, connector_impl=MySQLdb): + def connect(self, connector_impl=mysql.connector): """Establish a connection to the database.""" u, p = secrets.db.epi diff --git a/src/acquisition/ecdc/ecdc_db_update.py b/src/acquisition/ecdc/ecdc_db_update.py index 70f4c3cee..84423c376 100644 --- a/src/acquisition/ecdc/ecdc_db_update.py +++ b/src/acquisition/ecdc/ecdc_db_update.py @@ -37,7 +37,7 @@ import tempfile # third party -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -48,7 +48,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -98,7 +98,7 @@ def update_from_file(issue, date, dir, test_mode=False): # Read ECDC data from CSVs and insert into (or update) the database. # database connection u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx, "ecdc_ili") print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/flusurv/flusurv_update.py b/src/acquisition/flusurv/flusurv_update.py index 17490fcb1..1aa8e9885 100644 --- a/src/acquisition/flusurv/flusurv_update.py +++ b/src/acquisition/flusurv/flusurv_update.py @@ -72,7 +72,7 @@ import argparse # third party -import MySQLdb +import mysql.connector # first party from delphi.epidata.acquisition.flusurv import flusurv @@ -105,7 +105,7 @@ def update(issue, location_name, test_mode=False): # connect to the database u, p = secrets.db.epi - cnx = MySQLdb.connect(host=secrets.db.host, user=u, password=p, database="epidata") + cnx = mysql.connector.connect(host=secrets.db.host, user=u, password=p, database="epidata") cur = cnx.cursor() rows1 = get_rows(cur) print(f"rows before: {int(rows1)}") diff --git a/src/acquisition/fluview/fluview_notify.py b/src/acquisition/fluview/fluview_notify.py index e597d2098..3ed1a243f 100644 --- a/src/acquisition/fluview/fluview_notify.py +++ b/src/acquisition/fluview/fluview_notify.py @@ -25,7 +25,7 @@ import argparse # third party -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -46,7 +46,7 @@ # connect u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) cur = cnx.cursor() # get the last known issue from the automation table `variables` diff --git a/src/acquisition/fluview/fluview_update.py b/src/acquisition/fluview/fluview_update.py index f292c281d..defd01dad 100644 --- a/src/acquisition/fluview/fluview_update.py +++ b/src/acquisition/fluview/fluview_update.py @@ -121,7 +121,7 @@ import zipfile # third party -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -311,7 +311,7 @@ def update_from_file_clinical(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx, CL_TABLE) print(f"rows before: {int(rows1)}") insert = cnx.cursor() @@ -376,7 +376,7 @@ def update_from_file_public(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx, PHL_TABLE) print(f"rows before: {int(rows1)}") insert = cnx.cursor() @@ -445,7 +445,7 @@ def update_from_file(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx) print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/fluview/impute_missing_values.py b/src/acquisition/fluview/impute_missing_values.py index fc0d8d357..c795d9cce 100644 --- a/src/acquisition/fluview/impute_missing_values.py +++ b/src/acquisition/fluview/impute_missing_values.py @@ -49,7 +49,7 @@ import argparse # third party -import MySQLdb +import mysql.connector import numpy as np # first party @@ -135,7 +135,7 @@ class Sql: def connect(self): """Connect to the database.""" u, p = secrets.db.epi - self.cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) + self.cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) self.cur = self.cnx.cursor() def close(self, commit): diff --git a/src/acquisition/ght/ght_update.py b/src/acquisition/ght/ght_update.py index a58283d76..9e8d48d1d 100644 --- a/src/acquisition/ght/ght_update.py +++ b/src/acquisition/ght/ght_update.py @@ -70,7 +70,7 @@ import time # third party -import MySQLdb +import mysql.connector from apiclient.discovery import build # first party @@ -251,7 +251,7 @@ def update(locations, terms, first=None, last=None, countries=["US"]): # connect to the database u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/kcdc/kcdc_update.py b/src/acquisition/kcdc/kcdc_update.py index 1fae58c95..713b21f00 100644 --- a/src/acquisition/kcdc/kcdc_update.py +++ b/src/acquisition/kcdc/kcdc_update.py @@ -35,7 +35,7 @@ import requests # third party -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -45,7 +45,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -124,7 +124,7 @@ def get_kcdc_data(): def update_from_data(ews, ilis, date, issue, test_mode=False): u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx) print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/nidss/taiwan_update.py b/src/acquisition/nidss/taiwan_update.py index 2de4ab8b8..30d458481 100644 --- a/src/acquisition/nidss/taiwan_update.py +++ b/src/acquisition/nidss/taiwan_update.py @@ -77,7 +77,7 @@ import argparse # third party -import MySQLdb +import mysql.connector # first party from .taiwan_nidss import NIDSS @@ -105,7 +105,7 @@ def update(test_mode=False): # Database connection u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx) print(f"rows before (flu): {int(rows1[0])}") print(f"rows before (dengue): {int(rows1[1])}") diff --git a/src/acquisition/paho/paho_db_update.py b/src/acquisition/paho/paho_db_update.py index 0dfd08e40..b351d3ff2 100644 --- a/src/acquisition/paho/paho_db_update.py +++ b/src/acquisition/paho/paho_db_update.py @@ -54,7 +54,7 @@ import tempfile # third party -import MySQLdb +import mysql.connector import pycountry # first party @@ -66,7 +66,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -169,7 +169,7 @@ def update_from_file(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx, "paho_dengue") print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/quidel/quidel_update.py b/src/acquisition/quidel/quidel_update.py index a1ced5652..563cea898 100644 --- a/src/acquisition/quidel/quidel_update.py +++ b/src/acquisition/quidel/quidel_update.py @@ -39,7 +39,7 @@ import argparse # third party -import MySQLdb +import mysql.connector # first party from delphi.epidata.acquisition.quidel import quidel @@ -64,7 +64,7 @@ def update(locations, first=None, last=None, force_update=False, load_email=True qd_ts = quidel.measurement_to_ts(qd_measurements, 7, startweek=first, endweek=last) # connect to the database u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/twtr/twitter_update.py b/src/acquisition/twtr/twitter_update.py index c89681af2..80a023f19 100644 --- a/src/acquisition/twtr/twitter_update.py +++ b/src/acquisition/twtr/twitter_update.py @@ -52,7 +52,7 @@ """ # third party -import MySQLdb +import mysql.connector # first party from .healthtweets import HealthTweets @@ -62,7 +62,7 @@ def run(): # connect to the database u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/wiki/wiki_extract.py b/src/acquisition/wiki/wiki_extract.py index 5f886f2e5..718a64c20 100644 --- a/src/acquisition/wiki/wiki_extract.py +++ b/src/acquisition/wiki/wiki_extract.py @@ -28,7 +28,7 @@ import json # third party -import MySQLdb +import mysql.connector # first party import delphi.operations.secrets as secrets @@ -67,7 +67,7 @@ def get_timestamp(name): def run(job_limit=100): # connect to the database u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # # Some preparation for utf-8, and it is a temporary trick solution. The real solution should change those char set and collation encoding to utf8 permanently diff --git a/src/acquisition/wiki/wiki_update.py b/src/acquisition/wiki/wiki_update.py index 667ac9b46..a9f240629 100644 --- a/src/acquisition/wiki/wiki_update.py +++ b/src/acquisition/wiki/wiki_update.py @@ -24,7 +24,7 @@ from datetime import datetime, timedelta # third party -import MySQLdb +import mysql.connector import requests # first party @@ -90,7 +90,7 @@ def get_manifest(year, month, optional=False): def run(): # connect to the database u, p = secrets.db.epi - cnx = MySQLdb.connect(user=u, password=p, database="epidata") + cnx = mysql.connector.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # get the most recent job in wiki_raw diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 67fda472d..47c9f68e4 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -2,7 +2,7 @@ import unittest # third party -import MySQLdb +import mysql.connector # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -33,7 +33,7 @@ def setUp(self) -> None: """Perform per-test setup.""" # connect to the `epidata` database - cnx = MySQLdb.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") cur = cnx.cursor() cur.execute("DELETE FROM `api_user`") diff --git a/src/maintenance/remove_outdated_keys.py b/src/maintenance/remove_outdated_keys.py index 263565cae..56fea0cf9 100644 --- a/src/maintenance/remove_outdated_keys.py +++ b/src/maintenance/remove_outdated_keys.py @@ -2,7 +2,7 @@ from smtplib import SMTP import delphi.operations.secrets as secrets -import MySQLdb +import mysql.connector from delphi.epidata.server._config import API_KEY_REGISTRATION_FORM_LINK_LOCAL ApiUserRecord = namedtuple("APIUserRecord", ("api_key", "email", "date_diff")) @@ -51,7 +51,7 @@ def send_notification(to_addr, alert=True): def main(): u, p = secrets.db.epi - cnx = MySQLdb.connect(database="epidata", user=u, password=p, host=secrets.db.host) + cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() outdated_keys_list = get_old_keys(cur) for item in outdated_keys_list: diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index 75ec2fe9b..b7f1048f5 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -5,7 +5,7 @@ import sys import time import datetime -import MySQLdb +import mysql.connector import pandas as pd from dataclasses import dataclass @@ -62,7 +62,7 @@ class Database: STATUS_TABLE_NAME = 'dashboard_signal_status' COVERAGE_TABLE_NAME = 'dashboard_signal_coverage' - def __init__(self, connector_impl=MySQLdb): + def __init__(self, connector_impl=mysql.connector): """Establish a connection to the database.""" u, p = secrets.db.epi @@ -264,13 +264,13 @@ def main(args): try: database.write_status(signal_status_list) logger.info("Wrote status.", rowcount=database.rowcount()) - except MySQLdb.MySQLError as exception: + except mysql.connector.Error as exception: logger.exception(exception) try: database.write_coverage(coverage_list) logger.info("Wrote coverage.", rowcount=database.rowcount()) - except MySQLdb.MySQLError as exception: + except mysql.connector.Error as exception: logger.exception(exception) logger.info( diff --git a/src/maintenance/update_last_usage.py b/src/maintenance/update_last_usage.py index dd9a52a58..85596778a 100644 --- a/src/maintenance/update_last_usage.py +++ b/src/maintenance/update_last_usage.py @@ -2,7 +2,7 @@ from datetime import datetime as dtime import delphi.operations.secrets as secrets -import MySQLdb +import mysql.connector import redis REDIS_HOST = os.environ.get("REDIS_HOST", "delphi_redis") @@ -13,7 +13,7 @@ def main(): redis_cli = redis.Redis(host=REDIS_HOST, password=REDIS_PASSWORD, decode_responses=True) u, p = secrets.db.epi - cnx = MySQLdb.connect(database="epidata", user=u, password=p, host=secrets.db.host) + cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() redis_keys = redis_cli.keys(pattern=LAST_USED_KEY_PATTERN) From bbd822d4f2eef301b2ae2075820bf8645ad0466b Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 19:43:28 +0200 Subject: [PATCH 04/11] Replace in tests --- .../acquisition/covidcast/test_covidcast_meta_caching.py | 6 +++--- integrations/acquisition/covidcast/test_csv_uploading.py | 6 +++--- integrations/server/test_covidcast.py | 4 ++-- integrations/server/test_covidcast_meta.py | 4 ++-- integrations/server/test_fluview.py | 4 ++-- integrations/server/test_fluview_meta.py | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 30b6bbcf2..2ee59465f 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -5,7 +5,7 @@ import unittest # third party -import mysql.connector +import MySQLdb import requests # first party @@ -32,7 +32,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', @@ -58,7 +58,7 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') - epidata_cnx = mysql.connector.connect( + epidata_cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index dab35f414..968484762 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -7,7 +7,7 @@ import argparse # third party -import mysql.connector +import MySQLdb import pandas as pd import numpy as np @@ -28,7 +28,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', @@ -55,7 +55,7 @@ def setUp(self): secrets.db.host = 'delphi_database_epidata' secrets.db.epi = ('user', 'pass') - epidata_cnx = mysql.connector.connect( + epidata_cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 8b7fc6f52..c048f8a1f 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -5,7 +5,7 @@ import unittest # third party -import mysql.connector +import MySQLdb # first party from delphi_utils import Nans @@ -369,7 +369,7 @@ def test_unique_key_constraint(self): row = self._insert_placeholder_set_one() # fail to insert different placeholder data under the same key - with self.assertRaises(mysql.connector.errors.IntegrityError): + with self.assertRaises(MySQLdb.errors.IntegrityError): self._insert_placeholder_set_one() # succeed to insert different placeholder data under a different time_type diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index 857422a41..7fba26cbb 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -4,7 +4,7 @@ import unittest # third party -import mysql.connector +import MySQLdb import requests #first party @@ -53,7 +53,7 @@ def localSetUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `covidcast` table - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index 48d9585fd..b6a7719c5 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -4,7 +4,7 @@ import unittest # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -25,7 +25,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `fluview` table - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 6f81c1859..2e23a994a 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -4,7 +4,7 @@ import unittest # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -25,7 +25,7 @@ def setUp(self): """Perform per-test setup.""" # connect to the `epidata` database and clear the `fluview` table - cnx = mysql.connector.connect( + cnx = MySQLdb.connect( user='user', password='pass', host='delphi_database_epidata', From 44580e7004aac8a72a9163ad1b60f8ef46f6d444 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 19:48:25 +0200 Subject: [PATCH 05/11] Replace in code --- src/acquisition/cdcp/cdc_dropbox_receiver.py | 4 ++-- src/acquisition/cdcp/cdc_extract.py | 4 ++-- src/acquisition/cdcp/cdc_upload.py | 4 ++-- src/acquisition/ecdc/ecdc_db_update.py | 6 +++--- src/acquisition/flusurv/flusurv_update.py | 4 ++-- src/acquisition/fluview/fluview_notify.py | 4 ++-- src/acquisition/fluview/fluview_update.py | 8 ++++---- src/acquisition/fluview/impute_missing_values.py | 4 ++-- src/acquisition/ght/ght_update.py | 4 ++-- src/acquisition/kcdc/kcdc_update.py | 6 +++--- src/acquisition/nidss/taiwan_update.py | 4 ++-- src/acquisition/paho/paho_db_update.py | 6 +++--- src/acquisition/quidel/quidel_update.py | 4 ++-- src/acquisition/twtr/twitter_update.py | 4 ++-- src/acquisition/wiki/wiki_extract.py | 4 ++-- src/acquisition/wiki/wiki_update.py | 4 ++-- src/common/integration_test_base_class.py | 4 ++-- src/maintenance/remove_outdated_keys.py | 4 ++-- src/maintenance/update_last_usage.py | 4 ++-- 19 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/acquisition/cdcp/cdc_dropbox_receiver.py b/src/acquisition/cdcp/cdc_dropbox_receiver.py index 4fa20368e..9713aa384 100644 --- a/src/acquisition/cdcp/cdc_dropbox_receiver.py +++ b/src/acquisition/cdcp/cdc_dropbox_receiver.py @@ -22,7 +22,7 @@ # third party import dropbox -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -47,7 +47,7 @@ def trigger_further_processing(): # connect u, p = secrets.db.auto - cnx = mysql.connector.connect(user=u, password=p, database="automation") + cnx = MySQLdb.connect(user=u, password=p, database="automation") cur = cnx.cursor() # add step "Process CDCP Data" to queue diff --git a/src/acquisition/cdcp/cdc_extract.py b/src/acquisition/cdcp/cdc_extract.py index 0d38e0bcc..13237c2da 100644 --- a/src/acquisition/cdcp/cdc_extract.py +++ b/src/acquisition/cdcp/cdc_extract.py @@ -66,7 +66,7 @@ import sys # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -154,7 +154,7 @@ def extract(first_week=None, last_week=None, test_mode=False): # connect u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # weeks to update diff --git a/src/acquisition/cdcp/cdc_upload.py b/src/acquisition/cdcp/cdc_upload.py index 0e191267b..e154f2e62 100644 --- a/src/acquisition/cdcp/cdc_upload.py +++ b/src/acquisition/cdcp/cdc_upload.py @@ -80,7 +80,7 @@ from zipfile import ZipFile # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -165,7 +165,7 @@ def upload(test_mode): # connect u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # insert (or update) table `cdc` diff --git a/src/acquisition/ecdc/ecdc_db_update.py b/src/acquisition/ecdc/ecdc_db_update.py index 84423c376..70f4c3cee 100644 --- a/src/acquisition/ecdc/ecdc_db_update.py +++ b/src/acquisition/ecdc/ecdc_db_update.py @@ -37,7 +37,7 @@ import tempfile # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -48,7 +48,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -98,7 +98,7 @@ def update_from_file(issue, date, dir, test_mode=False): # Read ECDC data from CSVs and insert into (or update) the database. # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx, "ecdc_ili") print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/flusurv/flusurv_update.py b/src/acquisition/flusurv/flusurv_update.py index 1aa8e9885..17490fcb1 100644 --- a/src/acquisition/flusurv/flusurv_update.py +++ b/src/acquisition/flusurv/flusurv_update.py @@ -72,7 +72,7 @@ import argparse # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.acquisition.flusurv import flusurv @@ -105,7 +105,7 @@ def update(issue, location_name, test_mode=False): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(host=secrets.db.host, user=u, password=p, database="epidata") + cnx = MySQLdb.connect(host=secrets.db.host, user=u, password=p, database="epidata") cur = cnx.cursor() rows1 = get_rows(cur) print(f"rows before: {int(rows1)}") diff --git a/src/acquisition/fluview/fluview_notify.py b/src/acquisition/fluview/fluview_notify.py index 3ed1a243f..e597d2098 100644 --- a/src/acquisition/fluview/fluview_notify.py +++ b/src/acquisition/fluview/fluview_notify.py @@ -25,7 +25,7 @@ import argparse # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -46,7 +46,7 @@ # connect u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) cur = cnx.cursor() # get the last known issue from the automation table `variables` diff --git a/src/acquisition/fluview/fluview_update.py b/src/acquisition/fluview/fluview_update.py index defd01dad..e498e1363 100644 --- a/src/acquisition/fluview/fluview_update.py +++ b/src/acquisition/fluview/fluview_update.py @@ -121,7 +121,7 @@ import zipfile # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -311,7 +311,7 @@ def update_from_file_clinical(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx, CL_TABLE) print(f"rows before: {int(rows1)}") insert = cnx.cursor() @@ -376,7 +376,7 @@ def update_from_file_public(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx, PHL_TABLE) print(f"rows before: {int(rows1)}") insert = cnx.cursor() @@ -445,7 +445,7 @@ def update_from_file(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx) print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/fluview/impute_missing_values.py b/src/acquisition/fluview/impute_missing_values.py index c795d9cce..fc0d8d357 100644 --- a/src/acquisition/fluview/impute_missing_values.py +++ b/src/acquisition/fluview/impute_missing_values.py @@ -49,7 +49,7 @@ import argparse # third party -import mysql.connector +import MySQLdb import numpy as np # first party @@ -135,7 +135,7 @@ class Sql: def connect(self): """Connect to the database.""" u, p = secrets.db.epi - self.cnx = mysql.connector.connect(user=u, password=p, database="epidata", host=secrets.db.host) + self.cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) self.cur = self.cnx.cursor() def close(self, commit): diff --git a/src/acquisition/ght/ght_update.py b/src/acquisition/ght/ght_update.py index 9e8d48d1d..a58283d76 100644 --- a/src/acquisition/ght/ght_update.py +++ b/src/acquisition/ght/ght_update.py @@ -70,7 +70,7 @@ import time # third party -import mysql.connector +import MySQLdb from apiclient.discovery import build # first party @@ -251,7 +251,7 @@ def update(locations, terms, first=None, last=None, countries=["US"]): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/kcdc/kcdc_update.py b/src/acquisition/kcdc/kcdc_update.py index 713b21f00..1fae58c95 100644 --- a/src/acquisition/kcdc/kcdc_update.py +++ b/src/acquisition/kcdc/kcdc_update.py @@ -35,7 +35,7 @@ import requests # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -45,7 +45,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -124,7 +124,7 @@ def get_kcdc_data(): def update_from_data(ews, ilis, date, issue, test_mode=False): u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx) print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/nidss/taiwan_update.py b/src/acquisition/nidss/taiwan_update.py index 30d458481..2de4ab8b8 100644 --- a/src/acquisition/nidss/taiwan_update.py +++ b/src/acquisition/nidss/taiwan_update.py @@ -77,7 +77,7 @@ import argparse # third party -import mysql.connector +import MySQLdb # first party from .taiwan_nidss import NIDSS @@ -105,7 +105,7 @@ def update(test_mode=False): # Database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx) print(f"rows before (flu): {int(rows1[0])}") print(f"rows before (dengue): {int(rows1[1])}") diff --git a/src/acquisition/paho/paho_db_update.py b/src/acquisition/paho/paho_db_update.py index b351d3ff2..0dfd08e40 100644 --- a/src/acquisition/paho/paho_db_update.py +++ b/src/acquisition/paho/paho_db_update.py @@ -54,7 +54,7 @@ import tempfile # third party -import mysql.connector +import MySQLdb import pycountry # first party @@ -66,7 +66,7 @@ def ensure_tables_exist(): (u, p) = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") try: cursor = cnx.cursor() cursor.execute( @@ -169,7 +169,7 @@ def update_from_file(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") rows1 = get_rows(cnx, "paho_dengue") print(f"rows before: {int(rows1)}") insert = cnx.cursor() diff --git a/src/acquisition/quidel/quidel_update.py b/src/acquisition/quidel/quidel_update.py index 563cea898..a1ced5652 100644 --- a/src/acquisition/quidel/quidel_update.py +++ b/src/acquisition/quidel/quidel_update.py @@ -39,7 +39,7 @@ import argparse # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.acquisition.quidel import quidel @@ -64,7 +64,7 @@ def update(locations, first=None, last=None, force_update=False, load_email=True qd_ts = quidel.measurement_to_ts(qd_measurements, 7, startweek=first, endweek=last) # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/twtr/twitter_update.py b/src/acquisition/twtr/twitter_update.py index 80a023f19..c89681af2 100644 --- a/src/acquisition/twtr/twitter_update.py +++ b/src/acquisition/twtr/twitter_update.py @@ -52,7 +52,7 @@ """ # third party -import mysql.connector +import MySQLdb # first party from .healthtweets import HealthTweets @@ -62,7 +62,7 @@ def run(): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() def get_num_rows(): diff --git a/src/acquisition/wiki/wiki_extract.py b/src/acquisition/wiki/wiki_extract.py index 718a64c20..5f886f2e5 100644 --- a/src/acquisition/wiki/wiki_extract.py +++ b/src/acquisition/wiki/wiki_extract.py @@ -28,7 +28,7 @@ import json # third party -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -67,7 +67,7 @@ def get_timestamp(name): def run(job_limit=100): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # # Some preparation for utf-8, and it is a temporary trick solution. The real solution should change those char set and collation encoding to utf8 permanently diff --git a/src/acquisition/wiki/wiki_update.py b/src/acquisition/wiki/wiki_update.py index a9f240629..667ac9b46 100644 --- a/src/acquisition/wiki/wiki_update.py +++ b/src/acquisition/wiki/wiki_update.py @@ -24,7 +24,7 @@ from datetime import datetime, timedelta # third party -import mysql.connector +import MySQLdb import requests # first party @@ -90,7 +90,7 @@ def get_manifest(year, month, optional=False): def run(): # connect to the database u, p = secrets.db.epi - cnx = mysql.connector.connect(user=u, password=p, database="epidata") + cnx = MySQLdb.connect(user=u, password=p, database="epidata") cur = cnx.cursor() # get the most recent job in wiki_raw diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 47c9f68e4..67fda472d 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -2,7 +2,7 @@ import unittest # third party -import mysql.connector +import MySQLdb # first party from delphi.epidata.client.delphi_epidata import Epidata @@ -33,7 +33,7 @@ def setUp(self) -> None: """Perform per-test setup.""" # connect to the `epidata` database - cnx = mysql.connector.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") + cnx = MySQLdb.connect(user="user", password="pass", host="delphi_database_epidata", database="epidata") cur = cnx.cursor() cur.execute("DELETE FROM `api_user`") diff --git a/src/maintenance/remove_outdated_keys.py b/src/maintenance/remove_outdated_keys.py index 56fea0cf9..263565cae 100644 --- a/src/maintenance/remove_outdated_keys.py +++ b/src/maintenance/remove_outdated_keys.py @@ -2,7 +2,7 @@ from smtplib import SMTP import delphi.operations.secrets as secrets -import mysql.connector +import MySQLdb from delphi.epidata.server._config import API_KEY_REGISTRATION_FORM_LINK_LOCAL ApiUserRecord = namedtuple("APIUserRecord", ("api_key", "email", "date_diff")) @@ -51,7 +51,7 @@ def send_notification(to_addr, alert=True): def main(): u, p = secrets.db.epi - cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) + cnx = MySQLdb.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() outdated_keys_list = get_old_keys(cur) for item in outdated_keys_list: diff --git a/src/maintenance/update_last_usage.py b/src/maintenance/update_last_usage.py index 85596778a..dd9a52a58 100644 --- a/src/maintenance/update_last_usage.py +++ b/src/maintenance/update_last_usage.py @@ -2,7 +2,7 @@ from datetime import datetime as dtime import delphi.operations.secrets as secrets -import mysql.connector +import MySQLdb import redis REDIS_HOST = os.environ.get("REDIS_HOST", "delphi_redis") @@ -13,7 +13,7 @@ def main(): redis_cli = redis.Redis(host=REDIS_HOST, password=REDIS_PASSWORD, decode_responses=True) u, p = secrets.db.epi - cnx = mysql.connector.connect(database="epidata", user=u, password=p, host=secrets.db.host) + cnx = MySQLdb.connect(database="epidata", user=u, password=p, host=secrets.db.host) cur = cnx.cursor() redis_keys = redis_cli.keys(pattern=LAST_USED_KEY_PATTERN) From 7f240fe505e414ec7457ee374bb4f934e7b9e41c Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 20:08:16 +0200 Subject: [PATCH 06/11] two more --- src/acquisition/covid_hosp/common/database.py | 13 +++++++--- src/maintenance/signal_dash_data_generator.py | 25 ++++++++++++------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/src/acquisition/covid_hosp/common/database.py b/src/acquisition/covid_hosp/common/database.py index efbdb6c45..a0870f67b 100644 --- a/src/acquisition/covid_hosp/common/database.py +++ b/src/acquisition/covid_hosp/common/database.py @@ -6,7 +6,7 @@ import math # third party -import mysql.connector +import MySQLdb import pandas as pd # first party @@ -60,7 +60,7 @@ def logger(database_class): @classmethod @contextmanager - def connect(database_class, mysql_connector_impl=mysql.connector): + def connect(database_class, mysql_connector_impl=None): """Connect to a database and provide the connection as a context manager. As long as the context manager exits normally, the connection's transaction @@ -73,7 +73,14 @@ def connect(database_class, mysql_connector_impl=mysql.connector): # connect to the database user, password = secrets.db.epi - connection = mysql_connector_impl.connect( + if mysql_connector_impl is not None: + connection = mysql_connector_impl.connect( + host=secrets.db.host, + user=user, + password=password, + database='epidata') + else: + connection = MySQLdb.connect( host=secrets.db.host, user=user, password=password, diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index b7f1048f5..e9c704803 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -5,7 +5,7 @@ import sys import time import datetime -import mysql.connector +import MySQLdb import pandas as pd from dataclasses import dataclass @@ -62,15 +62,22 @@ class Database: STATUS_TABLE_NAME = 'dashboard_signal_status' COVERAGE_TABLE_NAME = 'dashboard_signal_coverage' - def __init__(self, connector_impl=mysql.connector): + def __init__(self, connector_impl=None): """Establish a connection to the database.""" u, p = secrets.db.epi - self._connection = connector_impl.connect( - host=secrets.db.host, - user=u, - password=p, - database=Database.DATABASE_NAME) + if connector_impl is None: + self._connection = MySQLdb.connect( + host=secrets.db.host, + user=u, + password=p, + database=Database.DATABASE_NAME) + else: + self._connection = connector_impl.connect( + host=secrets.db.host, + user=u, + password=p, + database=Database.DATABASE_NAME) self._cursor = self._connection.cursor() def rowcount(self) -> int: @@ -264,13 +271,13 @@ def main(args): try: database.write_status(signal_status_list) logger.info("Wrote status.", rowcount=database.rowcount()) - except mysql.connector.Error as exception: + except MySQLdb.MySQLError as exception: logger.exception(exception) try: database.write_coverage(coverage_list) logger.info("Wrote coverage.", rowcount=database.rowcount()) - except mysql.connector.Error as exception: + except MySQLdb.MySQLError as exception: logger.exception(exception) logger.info( From cf459bdc7f1c863921754ef195a5bb370a355f64 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 21:07:17 +0200 Subject: [PATCH 07/11] covidcast --- integrations/acquisition/covidcast/test_db.py | 2 +- .../server/test_covidcast_endpoints.py | 2 -- src/acquisition/covidcast/database.py | 33 +++++++++++-------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/integrations/acquisition/covidcast/test_db.py b/integrations/acquisition/covidcast/test_db.py index 7b9d80770..a966a5804 100644 --- a/integrations/acquisition/covidcast/test_db.py +++ b/integrations/acquisition/covidcast/test_db.py @@ -23,7 +23,7 @@ def _find_matches_for_row(self, row): cur.execute(q) res = cur.fetchone() if res: - results[table] = dict(zip(cur.column_names, res)) + results[table] = dict(zip([desc[0] for desc in cur.description], res)) else: results[table] = None return results diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 206ed3c54..14dc148d5 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -294,7 +294,6 @@ def test_backfill(self): def test_meta(self): """Request a signal from the /meta endpoint.""" - num_rows = 10 rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="fb-survey", signal="smoothed_cli") for i in range(num_rows)] self._insert_rows(rows) @@ -343,7 +342,6 @@ def test_meta_restricted(self): # and fed by src/server/endpoints/covidcast_utils/db_sources.csv, but also surreptitiously augmened # by _load_data_signals() which attaches a list of signals to each source, # in turn fed by src/server/endpoints/covidcast_utils/db_signals.csv) - # insert data from two different sources, one restricted/protected (quidel), one not self._insert_rows([ CovidcastTestRow.make_default_row(source="quidel", signal="raw_pct_negative"), diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 871061b81..156b2e722 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -10,7 +10,7 @@ # third party import json -import mysql.connector +import MySQLdb # first party import delphi.operations.secrets as secrets @@ -38,16 +38,23 @@ class Database: # TODO: also consider that for composite key tuples, like short_comp_key and long_comp_key as used in delete_batch() - def connect(self, connector_impl=mysql.connector): + def connect(self, connector_impl=None): """Establish a connection to the database.""" u, p = secrets.db.epi self._connector_impl = connector_impl - self._connection = self._connector_impl.connect( - host=secrets.db.host, - user=u, - password=p, - database=Database.DATABASE_NAME) + if connector_impl is None: + self._connection = MySQLdb.connect( + host=secrets.db.host, + user=u, + password=p, + database=Database.DATABASE_NAME) + else: + self._connection = self._connector_impl.connect( + host=secrets.db.host, + user=u, + password=p, + database=Database.DATABASE_NAME) self._cursor = self._connection.cursor() def commit(self): @@ -71,8 +78,7 @@ def disconnect(self, commit): def count_all_load_rows(self): self._cursor.execute(f'SELECT count(1) FROM `{self.load_table}`') - for (num,) in self._cursor: - return num + return self._cursor.fetchone()[0] def _reset_load_table_ai_counter(self): """Corrects the AUTO_INCREMENT counter in the load table. @@ -101,7 +107,7 @@ def do_analyze(self): f'''ANALYZE TABLE signal_dim, geo_dim, {self.load_table}, {self.history_table}, {self.latest_table}''') - output = [self._cursor.column_names] + self._cursor.fetchall() + output = [desc[0] for desc in self._cursor.description] + list(self._cursor.fetchall()) get_structured_logger('do_analyze').info("ANALYZE results", results=str(output)) def insert_or_update_bulk(self, cc_rows): @@ -456,8 +462,8 @@ def compute_covidcast_meta(self, table_name=None, n_threads=None): srcsigs = Queue() # multi-consumer threadsafe! sql = f'SELECT `source`, `signal` FROM `{table_name}` GROUP BY `source`, `signal` ORDER BY `source` ASC, `signal` ASC;' self._cursor.execute(sql) - for source, signal in self._cursor: - srcsigs.put((source, signal)) + for res in self._cursor.fetchall(): + srcsigs.put((res[0], res[1])) # source, signal inner_sql = f''' SELECT @@ -505,8 +511,9 @@ def worker(): logger.info("starting pair", thread=name, pair=f"({source}, {signal})") w_cursor.execute(inner_sql, (source, signal)) with meta_lock: + # Create a dictionary of column names (from cursor.description) & values meta.extend(list( - dict(zip(w_cursor.column_names, x)) for x in w_cursor + dict(zip([desc[0] for desc in w_cursor.description], x)) for x in w_cursor )) srcsigs.task_done() except Empty: From 03cb5b45578da1c360b31fe9904fc19bcc8d64a6 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 21:09:33 +0200 Subject: [PATCH 08/11] final --- requirements.dev.txt | 2 -- src/acquisition/covid_hosp/common/database.py | 19 ++++++------------- src/acquisition/covidcast/database.py | 19 ++++++------------- src/maintenance/signal_dash_data_generator.py | 19 ++++++------------- 4 files changed, 18 insertions(+), 41 deletions(-) diff --git a/requirements.dev.txt b/requirements.dev.txt index c1e3358c2..3f0d4f2f2 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -10,10 +10,8 @@ invoke>=1.4.1 lxml==4.9.1 matplotlib==3.6.2 mypy>=0.790 -mysql-connector-python==8.2.0 numpy==1.22.4 pycountry==22.3.5 -pymysql==1.1.0 pytest==7.2.0 pytest-check==1.3.0 sas7bdat==2.2.3 diff --git a/src/acquisition/covid_hosp/common/database.py b/src/acquisition/covid_hosp/common/database.py index a0870f67b..57286c831 100644 --- a/src/acquisition/covid_hosp/common/database.py +++ b/src/acquisition/covid_hosp/common/database.py @@ -60,7 +60,7 @@ def logger(database_class): @classmethod @contextmanager - def connect(database_class, mysql_connector_impl=None): + def connect(database_class, mysql_connector_impl=MySQLdb): """Connect to a database and provide the connection as a context manager. As long as the context manager exits normally, the connection's transaction @@ -73,18 +73,11 @@ def connect(database_class, mysql_connector_impl=None): # connect to the database user, password = secrets.db.epi - if mysql_connector_impl is not None: - connection = mysql_connector_impl.connect( - host=secrets.db.host, - user=user, - password=password, - database='epidata') - else: - connection = MySQLdb.connect( - host=secrets.db.host, - user=user, - password=password, - database='epidata') + connection = mysql_connector_impl.connect( + host=secrets.db.host, + user=user, + password=password, + database='epidata') try: # provide the connection to the context manager diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 156b2e722..9a6ea8b6a 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -38,23 +38,16 @@ class Database: # TODO: also consider that for composite key tuples, like short_comp_key and long_comp_key as used in delete_batch() - def connect(self, connector_impl=None): + def connect(self, connector_impl=MySQLdb): """Establish a connection to the database.""" u, p = secrets.db.epi self._connector_impl = connector_impl - if connector_impl is None: - self._connection = MySQLdb.connect( - host=secrets.db.host, - user=u, - password=p, - database=Database.DATABASE_NAME) - else: - self._connection = self._connector_impl.connect( - host=secrets.db.host, - user=u, - password=p, - database=Database.DATABASE_NAME) + self._connection = self._connector_impl.connect( + host=secrets.db.host, + user=u, + password=p, + database=Database.DATABASE_NAME) self._cursor = self._connection.cursor() def commit(self): diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index e9c704803..75ec2fe9b 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -62,22 +62,15 @@ class Database: STATUS_TABLE_NAME = 'dashboard_signal_status' COVERAGE_TABLE_NAME = 'dashboard_signal_coverage' - def __init__(self, connector_impl=None): + def __init__(self, connector_impl=MySQLdb): """Establish a connection to the database.""" u, p = secrets.db.epi - if connector_impl is None: - self._connection = MySQLdb.connect( - host=secrets.db.host, - user=u, - password=p, - database=Database.DATABASE_NAME) - else: - self._connection = connector_impl.connect( - host=secrets.db.host, - user=u, - password=p, - database=Database.DATABASE_NAME) + self._connection = connector_impl.connect( + host=secrets.db.host, + user=u, + password=p, + database=Database.DATABASE_NAME) self._cursor = self._connection.cursor() def rowcount(self) -> int: From f64688aa28ea7bb4ab7b1c265c8aa6daade99d1b Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 21:12:36 +0200 Subject: [PATCH 09/11] cleanup --- integrations/acquisition/covidcast/test_db.py | 1 + integrations/server/test_covidcast_endpoints.py | 2 ++ src/acquisition/covid_hosp/common/database.py | 8 ++++---- src/acquisition/covidcast/database.py | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/integrations/acquisition/covidcast/test_db.py b/integrations/acquisition/covidcast/test_db.py index a966a5804..35d344e15 100644 --- a/integrations/acquisition/covidcast/test_db.py +++ b/integrations/acquisition/covidcast/test_db.py @@ -23,6 +23,7 @@ def _find_matches_for_row(self, row): cur.execute(q) res = cur.fetchone() if res: + # Create a dictionary of column names (from cursor.description) & values results[table] = dict(zip([desc[0] for desc in cur.description], res)) else: results[table] = None diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 14dc148d5..65657ca4f 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -294,6 +294,7 @@ def test_backfill(self): def test_meta(self): """Request a signal from the /meta endpoint.""" + num_rows = 10 rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="fb-survey", signal="smoothed_cli") for i in range(num_rows)] self._insert_rows(rows) @@ -343,6 +344,7 @@ def test_meta_restricted(self): # by _load_data_signals() which attaches a list of signals to each source, # in turn fed by src/server/endpoints/covidcast_utils/db_signals.csv) # insert data from two different sources, one restricted/protected (quidel), one not + self._insert_rows([ CovidcastTestRow.make_default_row(source="quidel", signal="raw_pct_negative"), CovidcastTestRow.make_default_row(source="hhs", signal="confirmed_admissions_covid_1d") diff --git a/src/acquisition/covid_hosp/common/database.py b/src/acquisition/covid_hosp/common/database.py index 57286c831..1c79c37c5 100644 --- a/src/acquisition/covid_hosp/common/database.py +++ b/src/acquisition/covid_hosp/common/database.py @@ -74,10 +74,10 @@ def connect(database_class, mysql_connector_impl=MySQLdb): # connect to the database user, password = secrets.db.epi connection = mysql_connector_impl.connect( - host=secrets.db.host, - user=user, - password=password, - database='epidata') + host=secrets.db.host, + user=user, + password=password, + database='epidata') try: # provide the connection to the context manager diff --git a/src/acquisition/covidcast/database.py b/src/acquisition/covidcast/database.py index 9a6ea8b6a..860718448 100644 --- a/src/acquisition/covidcast/database.py +++ b/src/acquisition/covidcast/database.py @@ -100,6 +100,7 @@ def do_analyze(self): f'''ANALYZE TABLE signal_dim, geo_dim, {self.load_table}, {self.history_table}, {self.latest_table}''') + # Append lists of column names (from cursor.description) & values output = [desc[0] for desc in self._cursor.description] + list(self._cursor.fetchall()) get_structured_logger('do_analyze').info("ANALYZE results", results=str(output)) From 73c2d604ec00da8aa691da9f9309cd1a9ca560d2 Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Wed, 17 Jan 2024 21:33:06 +0200 Subject: [PATCH 10/11] Bump SQLAlchemy --- requirements.api.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.api.txt b/requirements.api.txt index fe66b0468..c0f51d837 100644 --- a/requirements.api.txt +++ b/requirements.api.txt @@ -13,7 +13,7 @@ redis==3.5.3 requests==2.31.0 scipy==1.10.0 sentry-sdk[flask] -SQLAlchemy==1.4.40 +SQLAlchemy==1.4.51 structlog==22.1.0 tenacity==7.0.0 typing-extensions From 596b8f642b21d919bcc30e4212f51e195383627d Mon Sep 17 00:00:00 2001 From: Rostyslav Zatserkovnyi Date: Tue, 23 Jan 2024 19:58:36 +0200 Subject: [PATCH 11/11] Fix details --- integrations/server/test_covidcast_endpoints.py | 2 +- src/acquisition/fluview/fluview_update.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 65657ca4f..206ed3c54 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -343,8 +343,8 @@ def test_meta_restricted(self): # and fed by src/server/endpoints/covidcast_utils/db_sources.csv, but also surreptitiously augmened # by _load_data_signals() which attaches a list of signals to each source, # in turn fed by src/server/endpoints/covidcast_utils/db_signals.csv) - # insert data from two different sources, one restricted/protected (quidel), one not + # insert data from two different sources, one restricted/protected (quidel), one not self._insert_rows([ CovidcastTestRow.make_default_row(source="quidel", signal="raw_pct_negative"), CovidcastTestRow.make_default_row(source="hhs", signal="confirmed_admissions_covid_1d") diff --git a/src/acquisition/fluview/fluview_update.py b/src/acquisition/fluview/fluview_update.py index e498e1363..f292c281d 100644 --- a/src/acquisition/fluview/fluview_update.py +++ b/src/acquisition/fluview/fluview_update.py @@ -376,7 +376,7 @@ def update_from_file_public(issue, date, filename, test_mode=False): # database connection u, p = secrets.db.epi - cnx = MySQLdb(user=u, password=p, database="epidata", host=secrets.db.host) + cnx = MySQLdb.connect(user=u, password=p, database="epidata", host=secrets.db.host) rows1 = get_rows(cnx, PHL_TABLE) print(f"rows before: {int(rows1)}") insert = cnx.cursor()