Skip to content

Commit

Permalink
Bulk update to sync with local changes
Browse files Browse the repository at this point in the history
  • Loading branch information
bahadirbasaran committed Sep 3, 2024
1 parent f18bf57 commit 28c526a
Show file tree
Hide file tree
Showing 26 changed files with 16,319 additions and 532 deletions.
21 changes: 15 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,21 @@ mkdir /teststat
cd <YOUR_DEV_ROOT>
# Checkout the repository to TESTstat
git clone https://github.com/bahadirbasaran/TESTstat.git
git clone https://gitlab.ripe.net/rnd/teststat.git
brew install [email protected]
```

## Workflow for virtual environment
```
# Create a virtualenv with the homebrew python
python3 -m venv venv3
python3.9 -m venv venv
# Activate the virtualenv
source venv3/bin/activate
source venv/bin/activate
# Install the packages inside virtualenv
pip install -r requirements.txt
pip install -r requirements/gui.txt
# Run the app
python main.py
Expand All @@ -39,10 +41,17 @@ brew install pyqt@5
source teststat_venv/bin/activate
# Install the packages inside virtualenv
pip install -r requirements_M1.txt
pip install -r requirements/gui_M1.txt
# Run the app
python main.py
```

## Containerized workflow

The GitLab Docker [build image](.gitlab/Dockerfile) installs a Python 3.9 virtual environment with all requirements and can be used locally.

```sh
docker build -t teststat -f .gitlab/Dockerfile .
docker run -v "$(pwd)":/src -it teststat
```
30 changes: 21 additions & 9 deletions core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,9 @@


# Application-wide definitions
BATCH_SIZE = 100

BATCH_SIZE = 20
MATTERMOST_URL = "https://mattermost.ripe.net/hooks/6xp8tt93i3fwde5d43jegsxi8a"
MATTERMOST_CHANNEL = "ripestat-teststat"
SLACK_URL = "https://hooks.slack.com/services/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"

ALL = "All following are True"
ANY = "At least one of following is True"
Expand Down Expand Up @@ -550,6 +549,15 @@ class ParamsCommons():
}
},

"dns-blocklists": {
"data_call_name": "DNS Blocklists",
"required_params": ["resource"],
"optional_params": [],
"output_params": {
"blocklists": {}
}
},

"example-resources": {
"data_call_name": "Example Resources",
"required_params": [],
Expand Down Expand Up @@ -716,7 +724,6 @@ class ParamsCommons():
}
},

# cannot obtain measurements
"meternet-bandwidth-measurements": {
"data_call_name": "Meter.net Bandwidth Measurements",
"required_params": ["resource"],
Expand All @@ -738,7 +745,7 @@ class ParamsCommons():
"resource": []
}
},
# in maintenance

"mlab-activity-count": {
"data_call_name": "Mlab Activity Count",
"required_params": ["resource"],
Expand All @@ -751,7 +758,7 @@ class ParamsCommons():
"resource": []
}
},
# in maintenance

"mlab-bandwidth": {
"data_call_name": "Mlab Bandwidth",
"required_params": ["resource"],
Expand All @@ -763,7 +770,7 @@ class ParamsCommons():
"resource": []
}
},
# in maintenance

"mlab-clients": {
"data_call_name": "Mlab Clients",
"required_params": ["resource"],
Expand Down Expand Up @@ -1073,7 +1080,8 @@ class ParamsCommons():
}
}
},
# how to optimize the keys here?

#TODO how to optimize the keys here?
"ris-peers": {
"data_call_name": "RIS Peers",
"required_params": [],
Expand Down Expand Up @@ -1345,7 +1353,7 @@ class ParamsCommons():
"rpki-history": {
"data_call_name": "RPKI History",
"required_params": ["resource"],
"optional_params": ["family", "resolution", "delegated"],
"optional_params": ["family", "resolution", "delegated", "include"],
"output_params": {
"timeseries": {
"asn": [ANY, NOT_EMPTY, MATCH],
Expand Down Expand Up @@ -1519,3 +1527,7 @@ class ParamsCommons():
}
}
}

DC_IN_MAINTENANCE = [
"blocklist",
]
78 changes: 78 additions & 0 deletions core/socket.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import socket


class INRDBSocket:
""" Generic socket wrapper to perform INRDB bulk queries """

def __init__(self,host,port):

self.buffer = ''
self.host = host
self.port = int(port)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

try:
self.sock.connect((self.host, self.port))
except socket.error as error_msg:
self.close_socket(
f"Could not initialize the socket {self.host}:{self.port} - {error_msg}"
)

self.send_line("-k")
ack = self.receive_line()
if ack != "% This is RIPE NCC's Routing Information Service":
self.close_socket(f"Connection handshake failed. Received: {ack}")
else:
last_line = ack
current_line = self.receive_line()

while last_line != '' and current_line != '':
last_line = current_line
current_line = self.receive_line()

def send_line(self, msg):

current_line = msg + '\n'
total_sent = 0
while total_sent < len(current_line):
sent = self.sock.send(current_line[total_sent:].encode('utf-8'))
if sent == 0:
self.close_socket(f"Socket connection broken: {self.host}:{self.port}")
total_sent = total_sent + sent

def receive_line(self):

index = self.buffer.find("\n")
if index == -1:
# No new line, but potentially still some bytes in buffer
# We still have data from previous call
current_line = self.buffer
new_line = False
else:
current_line = ''
chunk = self.buffer
new_line = True

while not new_line:
chunk = self.sock.recv(8192).decode("utf-8")

if chunk == '':
self.close_socket(f"Socket connection broken: {self.host}:{self.port}")
index = chunk.find("\n")
if index == -1:
current_line += chunk
else:
new_line = True

current_line += chunk[0:index]
self.buffer = chunk[index+1:len(chunk)]

return current_line

def close_socket(self, raise_exception_with_msg=""):

self.sock.shutdown(socket.SHUT_RDWR)
self.sock.close()

if raise_exception_with_msg:
raise RuntimeError(raise_exception_with_msg)
53 changes: 22 additions & 31 deletions core/teststat.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ def __init__(self, host, port=None, with_tls=True, gui=False):
host = host.lower().replace(' ', '')
self.session = aiohttp.ClientSession()
self.is_localhost = True if host == "127.0.0.1" or host == "localhost" else False
if self.is_localhost and not port:
port = "8000"

if self.is_localhost and port.isdecimal():
self.raw_query = f"http://127.0.0.1:{port}/data/"
Expand All @@ -39,49 +41,42 @@ async def run_test(
data_call,
test_input,
expected_output,
return_url=False,
return_data=False
return_output=False
):

if not self.is_localhost:
test_input += "&cache=ignore"

url = f"{self.raw_query}{data_call}/data.json?{test_input}"

timeout = aiohttp.ClientTimeout(total=15)
timeout = aiohttp.ClientTimeout(total=60)

try:
async with self.session.get(url, timeout=timeout) as response:
actual_output = await response.json()

except asyncio.TimeoutError:
if return_url:
return MessageEnum.TIMEOUT, url
return MessageEnum.TIMEOUT
return MessageEnum.TIMEOUT, url

# In case of receiving non-JSON response
# Hotfix for nginx returning non-JSON output from time to time.
except aiohttp.ContentTypeError:
if return_url:
return MessageEnum.BAD_GATEWAY, url
return MessageEnum.BAD_GATEWAY
# output = {
# "status_code": "500",
# "error": "Non-JSON response"
# }
output = {"status_code": "200"}
return output, url

except (aiohttp.ClientConnectorError, asyncio.CancelledError):
await self.session.close()
return MessageEnum.CONNECTION_ERROR, url

if return_url:
return MessageEnum.CONNECTION_ERROR, url
return MessageEnum.CONNECTION_ERROR

if return_data:
if return_url:
return actual_output, url
if return_output:
return actual_output

test_result = self.evaluate_result(data_call, actual_output, expected_output)

if return_url:
return test_result, url
return test_result
return test_result, url

def evaluate_result(self, data_call, test_output, expected_output):
"""
Expand Down Expand Up @@ -202,24 +197,17 @@ def _check_current_level(current_level, current_identifier):
nested_params = {}

# If status code is different than expected, directly return
expected_status_code = expected_output["status_code"]
if expected_status_code == "500" != str(test_output["status_code"]):
return failed_params

elif expected_status_code == "500" == str(test_output["status_code"]):
for message in test_output["messages"]:
if message[0] == "error":
return {"error": message[1].split("\n")[0]}
if (expected_output["status_code"] != str(test_output["status_code"])) or \
expected_output["status_code"] == "500" == str(test_output["status_code"]):

elif expected_status_code != str(test_output["status_code"]):
failed_params["status_code"] = str(test_output["status_code"])

for message in test_output["messages"]:
if message[0] == "error":
failed_params["error"] = message[1].split("\n")[0]
break
return failed_params

expected_output.pop("status_code")

# In some data call responses, 'data' is wrapped with 'results' key.
# Extract this key if in such case.
if "results" in test_output["data"]:
Expand All @@ -234,6 +222,9 @@ def _check_current_level(current_level, current_identifier):

for param, value in expected_output.items():

if param == "status_code":
continue

if param.split("->")[0] not in test_output["data"]:
failed_params[param] = "The output does not include this key!"
continue
Expand Down
Loading

0 comments on commit 28c526a

Please sign in to comment.