diff --git a/.gitignore b/.gitignore index c84997c..04004cb 100644 --- a/.gitignore +++ b/.gitignore @@ -69,9 +69,6 @@ instance/ # Scrapy stuff: .scrapy -# Sphinx documentation -docs/_build/ - # PyBuilder .pybuilder/ target/ @@ -139,4 +136,4 @@ dmypy.json cython_debug/ # notes, remove at merge -.notes/ \ No newline at end of file +.notes/ diff --git a/README.md b/README.md index 8afa26d..18f778b 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,77 @@ # Exercises for Distributed Systems -This repository contains a small framework written in python for emulating *asynchronous* and *synchronous* distributed systems. +This repository contains a small framework written in python for emulating *asynchronous* and *synchronous* distributed +systems. +## Install +The stepping emulator requires the following packages to run +* PyQt6 +* pynput +* cryptography + +These packages can be installed using `pip` as shown below: +```bash +pip install --user -r requirements.txt +``` + +These packages can be installed using `conda` as shown below: +```bash +conda env create -f environment.yml + +conda activate ds +``` ## General +A FAQ can be found [here](https://github.com/DEIS-Tools/DistributedExercisesAAU/wiki) + Exercises will be described later in this document. In general avoid changing any of the files in the `emulators` subdirectory. Instead, restrict your implementation to extending `emulators.Device` and `emulators.MessageStub`. -Your implementation/solution for, for instance, exersice 1 should go into the `exercises/exercise1.py` document. +Your implementation/solution for, for instance, exercise 1 should go into the `exercises/exercise1.py` document. I will provide new templates as the course progresses. You should be able to execute your solution to exercise 1 using the following lines: ```bash -python3.9 exercise_runner.py --lecture 1 --algorithm Gossip --type sync --devices 3 -python3.9 exercise_runner.py --lecture 1 --algorithm Gossip --type async --devices 3 +python exercise_runner.py --lecture 1 --algorithm Gossip --type sync --devices 3 +python exercise_runner.py --lecture 1 --algorithm Gossip --type async --devices 3 +python exercise_runner.py --lecture 1 --algorithm Gossip --type stepping --devices 3 ``` The first line will execute your implementation of the `Gossip` algorithm in a synchronous setting with three devices, while the second line will execute in an asynchronous setting. +The third line will execute your implementation in a synchronous setting, launching a GUI to visualize your +implementation, the setting can be adjusted during execution. For usage of the framework, see `exercises/demo.py` for a lightweight example. The example can be run with: ```bash -python3.9 exercise_runner.py --lecture 0 --algorithm PingPong --type async --devices 3 +python exercise_runner.py --lecture 0 --algorithm PingPong --type async --devices 3 +``` + +## Stepping emulator +The stepping emulator can be used to run the algorithm in steps where one message is sent or received for each step in +this emulator. The Stepping emulator can be controlled with the following keyboard input: +``` +space: Step a single time through messages +f: Fast-forward through messages +enter: Kill stepper daemon and finish algorithm +tab: Show all messages currently waiting tobe transmitted +s: Pick the next message waiting to be transmitted to transmit next +e: Toggle between sync and async emulation ``` +## GUI +The framework can also be launched with an interface by executing the following line: +``` +python exercise_runner_overlay.py +``` +Where your solution can be executed through this GUI. + +### Stepping emulator GUI +If the stepping emulator is chosen, the framework will launch with a GUI visualising some different aspects of your +algorithm, an example of the Stepping GUI is shown below: + +![](figures/stepping_gui.png) + ## Pull Requests If you have any extensions or improvements you are welcome to create a pull request. @@ -38,12 +86,15 @@ A number of persons initially know one distinct secret each. In each message, a person discloses all their secrets to the recipient. -These individuals can communicate only in pairs (no conference calls) but it is possible that different pairs of people talk concurrently. For all the tasks below you should consider the following two scenarios: +These individuals can communicate only in pairs (no conference calls) but it is possible that different pairs of people +talk concurrently. For all the tasks below you should consider the following two scenarios: - Scenario 1: a person may call any other person, thus the network is a total graph, - - Scenario 2: the persons are organized in a bi-directional circle, where the each person can only pass messages to the left and the right (use the modulo operator). + - Scenario 2: the persons are organized in a bi-directional circle, where each person can only pass messages to the + - left and the right (use the modulo operator). -In both scenarios you should use the `async` network, details of the differences between `sync` and `async` will be given in the third lecture. +In both scenarios you should use the `async` network, details of the differences between `sync` and `async` will be +given in the third lecture. Your tasks are as follows: @@ -53,39 +104,46 @@ Your tasks are as follows: - Is your solution optimal? And in what sense? ### NOTICE: -You can have several copies of the `Gossip` class, just give the class another name in the `exercise1.py` document, for instance `ImprovedGossip`. +You can have several copies of the `Gossip` class, just give the class another name in the `exercise1.py` document, for +instance `ImprovedGossip`. You should then be able to call the framework with your new class via: ```bash -python3.9 exercise_runner.py --lecture 1 --algorithm ImprovedGossip --type async --devices 3 +python exercise_runner.py --lecture 1 --algorithm ImprovedGossip --type async --devices 3 ``` # Exercise 2 -1. Implement the RIP protocol (fill in missing code in merge_tables), described in \[DS, fifth edition\] Page 115-118. -2. In the `__init__` of `RipCommunication`, create a ring topology (that is, set up who are the neighbors of each device). Consider a ring size of 10 devices. +1. Similarly to the first lecture, in the `__init__` of `RipCommunication`, create a ring topology (that is, set up who + are the neighbors of each device). +2. Implement the RIP protocol (fill in missing code in merge_tables), described in \[DS, fifth edition\] Page 115-118. + _(NOTICE: To run/debug the protocol, you must first implement the network topology described in "task 2.0" below.)_ +3. Now that you have a ring topology, consider a ring size of 10 devices. 1. How many messages are sent in total before the routing_tables of all nodes are synchronized? - 2. How can you "know" that the routing tables are complete and you can start using the network to route packets? Consider the general case of internet, and the specific case of our toy ring network. + 2. How can you "know" that the routing tables are complete and you can start using the network to route packets? + Consider the general case of internet, and the specific case of our toy ring network. 3. For the ring network, consider an approach similar to 1. ```python def routing_table_complete(self): - if len(self.routing_table) < self.number_of_devices()-1: + if len(self.routing_table) < self.number_of_devices-1: return False return True ``` - Does it work? Each routing table should believe it is completed just one. How many times the routing tables appear to be completed? + Does it work? Each routing table should believe it is completed just one row short. How many times do the + routing tables appear to be completed? 4. Try this other approach, which works better: 1. ```python def routing_table_complete(self): - if len(self.routing_table) < self.number_of_devices()-1: + if len(self.routing_table) < self.number_of_devices-1: return False for row in self.routing_table: (next_hop, distance) = self.routing_table[row] - if distance > (self.number_of_devices()/2): + if distance > (self.number_of_devices/2): return False return True ``` Is it realistic for a real network? -3. Send a `RoutableMessage` after the routing tables are ready. Consider the termination problem. Can a node quit right after receiving the `RoutingMessage` for itself? What happens to the rest of the nodes? -4. What happens, if a link has a negative cost? How many messages are sent before the `routing_tables` converge? +4. Send a `RoutableMessage` after the routing tables are ready. Consider the termination problem. Can a node quit right + after receiving the `RoutableMessage` for itself? What happens to the rest of the nodes? +5. What happens, if a link has a negative cost? How many messages are sent before the `routing_tables` converge? # Exercise 3 Please consult the moodle page, this exercise is not via this framework. @@ -97,10 +155,12 @@ implementing Suzuki-Kasami’s Mutex Algorithm. For all exercises today, you can use the `sync` network type - but most algorithms should work for `async` also. 1. Examine the algorithm - 1. Make a doodle on the blackboard/paper showing a few processes, their state, and messages exchanged. Use e.g. a sequence diagram. + 1. Make a doodle on the blackboard/paper showing a few processes, their state, and messages exchanged. Use e.g. a + sequence diagram. 2. Define the purpose of the vectors `_rn` and `_ln`. 2. Discuss the following situations - 1. Is it possible that a node receives a token request message after the corresponding request has been granted? Sketch a scenario. + 1. Is it possible that a node receives a token request message after the corresponding request has been granted? + Sketch a scenario. 2. How can a node know which nodes have ungranted requests? 3. How does the queue grow? @@ -115,14 +175,15 @@ For all exercises today, you can use the `sync` network type - but most algorith 3. Make it possible for new processes to join the ring. 5. Extracurricular exercise/challenge (only if you have nothing better to do over the weekend) - 1. Extend the mutex algorithm implementations s.t. the `do_work()` call starts an asynchronous process (e.g. a future) which later calls a `release()` method on the mutex classes. + 1. Extend the mutex algorithm implementations s.t. the `do_work()` call starts an asynchronous process (e.g. a + future) which later calls a `release()` method on the mutex classes. 2. Check that the algorithms still work, and modify where needed. 3. Submit a pull-request! # Exercise 5 -1. Identify two problems with IP-multicast - 1. What is a practical problem for IP-multicast? - 2. What is a theoretical problem for IP-multicast? +1. Identify two problems with Reliable Multicast over IP + 1. What is a practical problem for Reliable Multicast over IP? + 2. What is a theoretical problem for Reliable Multicast over IP? 2. Identify all the events in the following picture 1. Compute the lamport clocks for each event @@ -146,16 +207,19 @@ For all exercises today, you can use the `sync` network type - but most algorith 1. Hint: how (and when) do you identify a tie? # Exercise 6 -1. Study the pseudo-code in the slides (on moodle) and complete the implement of the `King` Algorithm in `exercise6.py` +1. Study the pseudo-code in the slides (on Moodle) and complete the implementation of the `King` Algorithm in + `exercise6.py` 1. How does the algorithm deal with a Byzantine king (try f=1, the first king is byzantine)? 2. Why does the algorithm satisfy Byzantine integrity? - 3. Sketch/discuss a modification your implementation s.t. the algorithm works in an `async` network, but looses its termination guarantee + 3. Sketch/discuss a modification of your implementation such that the algorithm works in an `async` network, but + looses its termination guarantee 1. What would happen with a Byzantine king? 2. What would happen with a slow king? 3. What about the combination of the above? -2. Bonus Exercise: Implement the Paxos algorithm in `exercise6.py`, see the pseudo-code on moodle (use the video for reference when in doubt) for the two interesting roles (proposer and acceptor). - 1. Identify messages send/received by each role +2. Bonus Exercise: Implement the Paxos algorithm in `exercise6.py`. See the pseudocode on Moodle (use the video for + reference when in doubt) for the two interesting roles (proposer and acceptor). + 1. Identify messages sent/received by each role 1. Investigate `PAXOSNetwork` 2. Implement each role but the learner 1. Assume that each device is both a `Proposer` and an `Acceptor` (the `Learner` is provided) @@ -163,82 +227,129 @@ For all exercises today, you can use the `sync` network type - but most algorith 3. Your job is to implement the missing functionality in `Proposer` and `Acceptor`, search for "TODO" 3. Demonstrate that your code works in an `async` environment 1. Try with a large number of devices (for instance 20 or 30) - 4. Discuss how you can use Paxos in "continued consensus" where you have to agree on the order of entries in a log-file + 4. Discuss how you can use Paxos in "continued consensus" where you have to agree on the order of entries in a + log-file # Exercise 7 -1. DS5ed 18.5, 18.13 -2. Sketch an architecture for the following three systems: A bulletin board (simple reddit), a bank, a version control system (e.g. GIT) - 1. Identify the system types - 2. Which replication type is suitable, and for which parts of the system +1. DS5ed exercises 18.10 and 18.13 +2. Sketch an architecture for the following three systems: A bulletin board (simple reddit), a bank, a version control + system (e.g. GIT) + 1. Identify the system types (with respect to CAP). + 2. Which replication type is suitable, and for which parts of the system? 3. If you go for a gossip solution, what is a suitable update frequency? 3. BONUS Exercise: Implement the Bully algorithm (DS 5ed, page 660) in `exercise7.py` 1. In which replication scheme is it useful? - 2. What is the "extra cost" of a new leader in replication? + 2. What is the "extra cost" of electing a new leader in replication? # Exercise 8 1. Compare GFS and Chubby, and identify use cases that are better for one or the other solution. 2. Consider the code in `exercise8.py`, which sketches GFS, and complete the "append record" implementation. 1. Take a look at how the GFS master is implemented, and how it translates file + chunk index to chunk handle - 2. Sketch a design for the "passive replication" solution of the GFS. Consider how many types of messages you need, when they should be sent, etc + 2. Sketch a design for the "passive replication" solution of the GFS. Consider how many types of messages you need, + when they should be sent, etc 3. Implement your solution, starting from the handler of RecordAppendReqMessage of the GfsChunkserver class 4. Try out your solution with a larger number of clients, to have more concurrent changes to the "file" -3. BONUS Exercise: Consider how to add shadow masters to the system. Clients and chunk servers will still interact with the first master to change the file system, but the shadow master can always work as read-only. +3. Implement fault tolerance mechanisms(Heartbeat Detection and Failover Mechanism) in the system: + 1. Heartbeat Detection: The GFS master should periodically receive heartbeats from each chunkserver to monitor their health. + 1. Modify the GfsChunkserver class to send periodic heartbeat messages to the GfsMaster. + 2. Update the GfsMaster class to maintain a list of active chunkservers based on received heartbeats. + 2. Failover Mechanism: When a chunkserver fails (i.e., stops sending heartbeats), the requests should be redirected to available replicas. + 1. Introduce a simple mechanism to simulate a chunkserver failing. + 2. Update the GfsMaster to detect a missing heartbeat and mark the corresponding chunkserver as failed. + 3. Ensure the client (GfsClient) can reroute operations to other replicas if a chunkserver has failed. +3. BONUS Exercise: Add shadow masters to the system. Clients and chunk servers will still interact with the first master + to change the file system, but the shadow master can take over if the master shuts down. NOTICE: To execute the code, issue for example: ```bash -python3.9 exercise_runner.py --lecture 8 --algorithm GfsNetwork --type async --devices 7 +python exercise_runner.py --lecture 8 --algorithm GfsNetwork --type async --devices 7 ``` # Exercise 9 -1. Consider the code in `exercise9.py`, which sketches MapReduce, and complete it. - 1. Unzip the file books.zip in ex9data/books - 2. The Master is pretty much complete, the same can be said for the client. Take a look at how the Master is supposed to interact with Mappers and Reducers - 3. Consider how to implement the Reducers. Take into account that we are simulating "local storage in the mappers" using memory - 4. Look for the TODOs, and implement your solution +1. How do MapReduce, Spark, and Pregel differ? Discuss the following: + 1. What are some major design and architectural differences in the three systems? + 2. What are the target use cases for the different system? When do they perform good or bad. + 3. What are trade-offs in terms of performance, scalability, and complexity for the three systems? +2. Consider the code in `exercise9.py`, which sketches MapReduce, and complete it. + 1. Unzip the file books.zip in ex9data/books. + 2. The Master is pretty much complete. The same can be said for the client. Take a look at how the Master is supposed + to interact with Mappers and Reducers. + 3. Consider how to implement the Reducers. Take into account that we are simulating "local storage in the mappers" + using memory. + 4. Look for the TODOs, and implement your solution. 5. Try to change the number of mappers and reducers, and look at the "performance". In particular, look at how many rounds are needed to complete the job with the "sync" simulator. -2. Compare MapReduce and Spark RDDs, and consider what it would change in terms of architecture, especially to supprt RDDs +3. Add simulation for stragglers (slow workers) + 1. Modify the `MapReduceWorker` `run` or `do_some_work` method to occasionally add a random delay for specific workers, which can rarely be very large. + 2. Modify the `MapReduceMaster`to track the progress of each worker and reassign uncompleted tasks if a worker takes too long. + 3. Discuss the real-world relevance of stragglers in distributed systems (e.g., slow network nodes, overloaded servers) and how they affect system throughput and overall performance. NOTICE: To execute the code, issue for example: ```bash -python3 exercise_runner.py --lecture 9 --algorithm MapReduceNetwork --type async --devices 6 +python exercise_runner.py --lecture 9 --algorithm MapReduceNetwork --type async --devices 6 ``` # Exercise 10 -1. There are "exercises" (actually, questions) on the moodle. I suggest to start with them. -2. Consider the code in `exercise10.py`, which sketches a blockchain similar to bitcoin. Consider that transactions are just strings and we will do no check on the transactions. I had to add a very "random" termination condition. I associate a miner to each client, the code will never stop if I have an odd number of devices. - 1. Take a look at the Block and the Blockchain (they are NOT devices) and consider how the blockchain is supposed to grow. - 2. Design the logic for when a miner sends a blockchain (with its new block) to another miner. What do you do when you receive a new block? What if a fork? Can it happen? How do you manage it to preserve the "longest chain" rule? - 3. Look for the TODOs, and implement your solution +1. There are "exercises" (actually, questions) on the Moodle page. I suggest to start with them. +2. Consider the code in `exercise10.py`, which sketches a blockchain similar to bitcoin. Consider that transactions are + just strings and we will do no check on the transactions. I had to add a very "random" termination condition. I + associate a miner to each client, the code will never stop if I have an odd number of devices. + 1. Take a look at the Block and the Blockchain classes (they are NOT devices) and consider how the blockchain is + supposed to grow. + 2. Design the logic for when a miner sends a blockchain (with its new block) to another miner. What do you do when + you receive a new block? What if there is a fork? Can it happen? How do you manage it to preserve the "longest + chain" rule? + 3. Look for the TODOs, and implement your solution. 4. Try the code for both sync and async devices. Does it work in both cases? +3. Consider the modified blockchain, we will simulate an attack that controls a percentage of the total computing power. The goal of the attacker is to create a fork of the blockchain that ignore the last 5 blocks, and eventually make the fork the dominant chain in the network. + 1. Think about how you can exploit a majority control of the network to alter the blockchain history by introducing a fork. + 1. How can the attackers ensure that their fork becomes the longest chain and eventually gets accepted by other miners? + 2. How do you deal with new transactions in both the original chain and the forked chain? + 2. Take a look at the existing BlockchainNetwork and BlockchainMiner classes. Modify these to implement your blockchain attack. + 1. Create the BlockchainAttacker, which misbehaves if _self.id() is lower than a fraction of the total number of nodes. + 2. Initiate a fork of the blockchain using the attackers. + 3. Make the attackers collude to ensure that the new fork surpasses the original. + 4. Observe how the other miners react to your fork and if they eventually switch to your chain. + 3. Play around with the number of attackers and discuss their impact on the network, can they create a new longest chain without majority control? + NOTICE: To execute the code, issue for example: ```bash -python3 exercise_runner.py --lecture 10 --algorithm BlockchainNetwork --type async --devices 4 +python exercise_runner.py --lecture 10 --algorithm BlockchainNetwork --type async --devices 4 ``` # Exercise 11 -1. There are "exercises" on the moodle. I suggest to start with them. -2. Consider the code in `exercise11.py`, which sets up the finger tables for chord nodes. I have a client, connected always to the same node, which issues some PUTs. +1. There are "exercises" on the Moodle page. I suggest to start with them. +2. Consider the code in `exercise11.py`, which sets up the finger tables for chord nodes. I have a client, connected + always to the same node, which issues some PUTs. 1. Take a look at how the finger tables are populated, but please use the slides, since the code can be quite cryptic. - 2. Design the logic for the routing process, thus: when do I end the routing process? Who should I send the message to, if I am not the destination? - 3. Look for the TODOs, and implement your solution + 2. Design the logic for the routing process, thus: when do I end the routing process? Who should I send the message + to, if I am not the destination? + 3. Look for the TODOs, and implement your solution. 4. If you have time, implement the JOIN process for device 1. NOTICE: To execute the code, issue for example: ```bash -python3 exercise_runner.py --lecture 11 --algorithm BlockchainNetwork --type async --devices 10 +python exercise_runner.py --lecture 11 --algorithm ChordNetwork --type async --devices 10 ``` # Exercise 12 -1. There are "exercises" on the moodle. I suggest to start with them. +1. There are "exercises" on the Moodle page. I suggest to start with them. 2. Consider the code in `exercise12.py`, which creates the topology for your IoT wireless network. The goal is to implement AODV. - 1. Please note that you can self.medium().send() messages only in the nodes in self.neighbors. This simulates a wireless network with limited range. - 2. Design the logic for the Route Request process. What can you use as a broadcast id? Design also the Route Reply, which should be much easier. + 1. Please note that you can `self.medium().send()` messages only to the nodes in `self.neighbors`. This simulates a + wireless network with limited range. + 2. Design the logic for the Route Request process. What can you use as a broadcast id? Design also the Route Reply + process, which should be much easier. 3. Look for the TODOs, and implement your solution. +3. Now consider a node going offline in the network. Simulate a node failure in the AODV-based network and observe how the network handles it. + 1. Discuss how you expect the network to handle the disconnection of a node. + 2. Implement logic that simulates a node failure for one of the nodes by disconnecting it from its neighbours. + 3. Observe the network's routing behaviour after the node fails. Does it match your expectations? + NOTICE: To execute the code, issue for example: ```bash -python3 exercise_runner.py --lecture 12 --algorithm AodvNode --type async --devices 10 +python exercise_runner.py --lecture 12 --algorithm AodvNode --type async --devices 10 ``` + diff --git a/conf.py b/conf.py new file mode 100644 index 0000000..72b2fb5 --- /dev/null +++ b/conf.py @@ -0,0 +1,14 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath(".")) + +extensions = [ + "sphinx.ext.autodoc", +] + +autodoc_modules = { + "Device": "./emulators/Device.py", + "Medium": "./emulators/Medium.py", + "MessageStub": "./emulators/MessageStub.py", +} diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..9cc0533 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,27 @@ +DistributedExercisesAAU docs +============================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +Device +------ + +.. automodule:: emulators.Device + :members: + :undoc-members: + +Medium +------ + +.. automodule:: emulators.Medium + :members: + :undoc-members: + +MessageStub +----------- + +.. automodule:: emulators.MessageStub + :members: + :undoc-members: diff --git a/emulators/AsyncEmulator.py b/emulators/AsyncEmulator.py index b17a4b3..47c9187 100644 --- a/emulators/AsyncEmulator.py +++ b/emulators/AsyncEmulator.py @@ -1,20 +1,27 @@ import copy import random -import threading import time -from threading import Lock from typing import Optional +from os import name from emulators.EmulatorStub import EmulatorStub from emulators.MessageStub import MessageStub +if name == "posix": + RESET = "\u001B[0m" + CYAN = "\u001B[36m" + GREEN = "\u001B[32m" +else: + RESET = "" + CYAN = "" + GREEN = "" -class AsyncEmulator(EmulatorStub): +class AsyncEmulator(EmulatorStub): def __init__(self, number_of_devices: int, kind): super().__init__(number_of_devices, kind) self._terminated = 0 - self._messages = {} + self._messages: dict[int, list[MessageStub]] = {} self._messages_sent = 0 def run(self): @@ -22,53 +29,67 @@ def run(self): self._start_threads() self._progress.release() - # make sure the round_lock is locked initially while True: time.sleep(0.1) self._progress.acquire() # check if everyone terminated - if self.all_terminated(): + if self.all_terminated: break self._progress.release() for t in self._threads: t.join() return - def queue(self, message: MessageStub): - self._progress.acquire() + def queue(self, message: MessageStub, stepper=False): + if not stepper: + self._progress.acquire() self._messages_sent += 1 - print(f'\tSend {message}') + print(f"\r\t{GREEN}Send{RESET} {message}") if message.destination not in self._messages: self._messages[message.destination] = [] - self._messages[message.destination].append(copy.deepcopy(message)) # avoid accidental memory sharing - random.shuffle(self._messages[message.destination]) # shuffle to emulate changes in order - time.sleep(random.uniform(0.01, 0.1)) # try to obfuscate delays and emulate network delays - self._progress.release() + self._messages[message.destination].append( + copy.deepcopy(message) + ) # avoid accidental memory sharing + random.shuffle( + self._messages[message.destination] + ) # shuffle to emulate changes in order + time.sleep( + random.uniform(0.01, 0.1) + ) # try to obfuscate delays and emulate network delays + if not stepper: + self._progress.release() - def dequeue(self, index: int) -> Optional[MessageStub]: - self._progress.acquire() + def dequeue(self, index: int, stepper=False) -> Optional[MessageStub]: + if not stepper: + self._progress.acquire() if index not in self._messages: - self._progress.release() + if not stepper: + self._progress.release() return None elif len(self._messages[index]) == 0: - self._progress.release() + if not stepper: + self._progress.release() return None else: m = self._messages[index].pop() - print(f'\tRecieve {m}') - self._progress.release() + print(f"\r\t{GREEN}Recieve{RESET} {m}") + if not stepper: + self._progress.release() return m def done(self, index: int): - time.sleep(random.uniform(0.01, 0.1)) # try to obfuscate delays and emulate network delays + time.sleep( + random.uniform(0.01, 0.1) + ) # try to obfuscate delays and emulate network delays return - def print_statistics(self): - print(f'\tTotal {self._messages_sent} messages') - print(f'\tAverage {self._messages_sent/len(self._devices)} messages/device') + print(f"\t{GREEN}Total{RESET} {self._messages_sent} messages") + print( + f"\t{GREEN}Average{RESET} {self._messages_sent/len(self._devices)} messages/device" + ) - def terminated(self, index:int): + def terminated(self, index: int): self._progress.acquire() self._terminated += 1 - self._progress.release() \ No newline at end of file + self._progress.release() diff --git a/emulators/Device.py b/emulators/Device.py index 841e27e..e93d559 100644 --- a/emulators/Device.py +++ b/emulators/Device.py @@ -5,24 +5,68 @@ class Device: + """ + Base class representing a device in a simulation. + + Args: + index (int): The unique identifier for the device. + number_of_devices (int): The total number of devices in the simulation. + medium (Medium): The communication medium used by the devices. + + Attributes: + _id (int): The unique identifier for the device. + _medium (Medium): The communication medium used by the device. + _number_of_devices (int): The total number of devices in the simulation. + _finished (bool): A flag indicating if the device has finished its task. + """ + def __init__(self, index: int, number_of_devices: int, medium: Medium): self._id = index self._medium = medium self._number_of_devices = number_of_devices + self._finished = False def run(self): + """ + Abstract method representing the main functionality of the device. + """ + raise NotImplementedError("You have to implement a run-method!") def print_result(self): + """ + Abstract method for the result printer + """ raise NotImplementedError("You have to implement a result printer!") + @property def index(self): + """ + The unique identifier for the device. + + Returns: + int: The unique identifier of the device. + """ return self._id + @property def number_of_devices(self): + """ + Get the total number of devices in the simulation. + + Returns: + int: The total number of devices. + """ return self._number_of_devices + @property def medium(self): + """ + Get the communication medium used by the device. + + Returns: + Medium: The communication medium. + """ return self._medium @@ -34,9 +78,12 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) self._has_work = False + @property def has_work(self) -> bool: # The random call emulates that a concurrent process asked for the - self._has_work = self._has_work or random.randint(0, self.number_of_devices()) == self.index() + self._has_work = ( + self._has_work or random.randint(0, self.number_of_devices) == self.index + ) return self._has_work def do_work(self): @@ -45,19 +92,19 @@ def do_work(self): # might require continued interaction. The "working" thread would then notify our Requester class back # when the mutex is done being used. self._lock.acquire() - print(f'Device {self.index()} has started working') + print(f"Device {self.index} has started working") self._concurrent_workers += 1 if self._concurrent_workers > 1: self._lock.release() raise Exception("More than one concurrent worker!") self._lock.release() - assert (self.has_work()) + assert self.has_work amount_of_work = random.randint(1, 4) for i in range(0, amount_of_work): - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() self._lock.acquire() - print(f'Device {self.index()} has ended working') + print(f"Device {self.index} has ended working") self._concurrent_workers -= 1 self._lock.release() self._has_work = False diff --git a/emulators/EmulatorStub.py b/emulators/EmulatorStub.py index 2da68c5..d68f043 100644 --- a/emulators/EmulatorStub.py +++ b/emulators/EmulatorStub.py @@ -6,7 +6,6 @@ class EmulatorStub: - def __init__(self, number_of_devices: int, kind): self._nids = number_of_devices self._devices = [] @@ -14,10 +13,12 @@ def __init__(self, number_of_devices: int, kind): self._media = [] self._progress = threading.Lock() - for index in self.ids(): + for index in self.ids: self._media.append(Medium(index, self)) self._devices.append(kind(index, number_of_devices, self._media[-1])) - self._threads.append(threading.Thread(target=self._run_thread, args=[index])) + self._threads.append( + threading.Thread(target=self._run_thread, args=[index]) + ) def _run_thread(self, index: int): self._devices[index].run() @@ -30,14 +31,15 @@ def _run_thread(self, index: int): def _start_threads(self): cpy = self._threads.copy() random.shuffle(cpy) - print('Starting Threads') + print("Starting Threads") for thread in cpy: thread.start() + @property def all_terminated(self) -> bool: - return all([not self._threads[x].is_alive() - for x in self.ids()]) + return all([not self._threads[x].is_alive() for x in self.ids]) + @property def ids(self): return range(0, self._nids) @@ -46,19 +48,19 @@ def print_result(self): d.print_result() def run(self): - raise NotImplementedError(f'Please contact the instructor') + raise NotImplementedError("Please contact the instructor") def queue(self, message: MessageStub): - raise NotImplementedError(f'Please contact the instructor') + raise NotImplementedError("Please contact the instructor") def dequeue(self, id) -> MessageStub: - raise NotImplementedError(f'Please contact the instructor') + raise NotImplementedError("Please contact the instructor") def done(self, id): - raise NotImplementedError(f'Please contact the instructor') + raise NotImplementedError("Please contact the instructor") def print_statistics(self): - raise NotImplementedError(f'Please contact the instructor') + raise NotImplementedError("Please contact the instructor") def terminated(self, index: int): - raise NotImplementedError(f'Please contact the instructor') + raise NotImplementedError("Please contact the instructor") diff --git a/emulators/Medium.py b/emulators/Medium.py index c86f764..4ec6d9a 100644 --- a/emulators/Medium.py +++ b/emulators/Medium.py @@ -1,9 +1,19 @@ -import threading - from emulators.MessageStub import MessageStub class Medium: + """ + Represents a communication medium in a simulation. + + Args: + index (int): The unique identifier for the medium. + emulator: The emulator object responsible for managing message queues. + + Attributes: + _id (int): The unique identifier for the medium. + _emulator: The emulator object associated with the medium. + """ + _id: int def __init__(self, index: int, emulator): @@ -11,12 +21,30 @@ def __init__(self, index: int, emulator): self._emulator = emulator def send(self, message: MessageStub): + """ + Send a message through the medium. + + Args: + message (MessageStub): The message to be sent. + """ self._emulator.queue(message) def receive(self) -> MessageStub: + """ + Receive a message from the medium. + + Returns: + MessageStub: The received message. + """ return self._emulator.dequeue(self._id) def receive_all(self) -> list[MessageStub]: + """ + Receive all available messages from the medium. + + Returns: + list[MessageStub]: A list of received messages. + """ messages = [] while True: message = self._emulator.dequeue(self._id) @@ -25,7 +53,19 @@ def receive_all(self) -> list[MessageStub]: messages.append(message) def wait_for_next_round(self): + """ + Wait for the next communication round. + + This method signals that the device is waiting for the next communication round to begin. + """ self._emulator.done(self._id) + @property def ids(self): - return self._emulator.ids() + """ + Get the unique identifier of the medium. + + Returns: + int: The unique identifier of the medium. + """ + return self._emulator.ids diff --git a/emulators/MessageStub.py b/emulators/MessageStub.py index 20108ed..55d2ae9 100644 --- a/emulators/MessageStub.py +++ b/emulators/MessageStub.py @@ -1,23 +1,62 @@ class MessageStub: + """ + Represents a message used in communication within a simulation. + + Attributes: + _source (int): The identifier of the message sender. + _destination (int): The identifier of the message receiver. + """ + _source: int _destination: int def __init__(self, sender_id: int, destination_id: int): + """ + Initialize a MessageStub instance. + + Args: + sender_id (int): The identifier of the message sender. + destination_id (int): The identifier of the message receiver. + """ self._source = sender_id self._destination = destination_id @property def destination(self) -> int: + """ + Get the identifier of the message's destination. + + Returns: + int: The identifier of the destination device. + """ return self._destination @property def source(self) -> int: + """ + Get the identifier of the message's source. + + Returns: + int: The identifier of the source device. + """ return self._source @destination.setter def destination(self, value): + """ + Set the identifier of the message's destination. + + Args: + value (int): The new identifier for the destination device. + """ self._destination = value @source.setter def source(self, value): + """ + Set the identifier of the message's source. + + Args: + value (int): The new identifier for the source device. + """ self._source = value diff --git a/emulators/SteppingEmulator.py b/emulators/SteppingEmulator.py index 425d4ab..71e4d24 100644 --- a/emulators/SteppingEmulator.py +++ b/emulators/SteppingEmulator.py @@ -1,92 +1,281 @@ import copy import random -import time -from emulators.AsyncEmulator import AsyncEmulator +from time import sleep from typing import Optional +from emulators.AsyncEmulator import AsyncEmulator +from .EmulatorStub import EmulatorStub +from emulators.SyncEmulator import SyncEmulator from emulators.MessageStub import MessageStub -from pynput import keyboard -from getpass import getpass #getpass to hide input, cleaner terminal -from threading import Thread #run getpass in seperate thread +from threading import ( + Barrier, + Lock, + BrokenBarrierError, +) # run getpass in seperate thread +from os import name + +if name == "posix": + RESET = "\u001B[0m" + CYAN = "\u001B[36m" + GREEN = "\u001B[32m" +else: + RESET = "" + CYAN = "" + GREEN = "" + +class SteppingEmulator(SyncEmulator, AsyncEmulator): + _single = False + last_action = "" + messages_received: list[MessageStub] = [] + messages_sent: list[MessageStub] = [] + keyheld = False + pick_device = -1 + pick_running = False + next_message = None + log = None + parent: EmulatorStub = AsyncEmulator -class SteppingEmulator(AsyncEmulator): - def __init__(self, number_of_devices: int, kind): #default init, add stuff here to run when creating object + def __init__( + self, number_of_devices: int, kind + ): # default init, add stuff here to run when creating object super().__init__(number_of_devices, kind) - self._stepper = Thread(target=lambda: getpass(""), daemon=True) - self._stepper.start() - self._stepping = True - self._single = False - self._keyheld = False - self.listener = keyboard.Listener(on_press=self.on_press, on_release=self.on_release) - self.listener.start() - msg = """ - keyboard input: - space: Step a single time through messages - f: Fast-forward through messages - enter: Kill stepper daemon and run as an async emulator + # self._stepper = Thread(target=lambda: getpass(""), daemon=True) + # self._stepper.start() + self.barrier = Barrier(parties=number_of_devices) + self.step_barrier = Barrier(parties=2) + self.is_stepping = True + self.input_lock = Lock() + # self.listener = keyboard.Listener(on_press=self._on_press, on_release=self._on_release) + # self.listener.start() + self.shell = self.prompt + self.messages_received: list[MessageStub] = [] + self.messages_sent: list[MessageStub] = [] + msg = f""" +{CYAN}Shell input:{RESET}: + {CYAN}step(press return){RESET}: Step a single time through messages + {CYAN}exit{RESET}: Finish the execution of the algorithm + {CYAN}queue{RESET}: Show all messages currently waiting to be transmitted + {CYAN}queue {RESET}: Show all messages currently waiting to be transmitted to a specific device + {CYAN}pick{RESET}: Pick the next message waiting to be transmitted to transmit next + {CYAN}swap{RESET}: Toggle between sync and async emulation """ print(msg) - def dequeue(self, index: int) -> Optional[MessageStub]: - #return super().dequeue(index) #uncomment to run as a normal async emulator (debug) self._progress.acquire() - if index not in self._messages: - self._progress.release() - return None - elif len(self._messages[index]) == 0: - self._progress.release() - return None + # print(f'thread {index} is in dequeue') + if self.next_message: + if not index == self.pick_device: + self.collectThread() + return self.dequeue(index) + else: + if self.parent == AsyncEmulator: + messages = self._messages + else: + messages = self._last_round_messages + result = messages[index].pop(messages[index].index(self.next_message)) + self.next_message = None + self.pick_device = -1 + self.barrier.reset() + print(f"\r\t{GREEN}Receive{RESET} {result}") + else: - if self._stepping and self._stepper.is_alive(): #first expression for printing a reasonable amount, second to hide user input - self._step("step?") - m = self._messages[index].pop() - print(f'\tRecieve {m}') - self._progress.release() - return m - + result = self.parent.dequeue(self, index, True) + + self.pick_running = False + + if result: + self.messages_received.append(result) + self.last_action = "receive" + if self.is_stepping: + self.step() + + self._progress.release() + return result + def queue(self, message: MessageStub): - #return super().queue(message) #uncomment to run as normal queue (debug) self._progress.acquire() - if self._stepping and self._stepper.is_alive(): - self._step("step?") - self._messages_sent += 1 - print(f'\tSend {message}') - if message.destination not in self._messages: - self._messages[message.destination] = [] - self._messages[message.destination].append(copy.deepcopy(message)) # avoid accidental memory sharing - random.shuffle(self._messages[message.destination]) # shuffle to emulate changes in order - time.sleep(random.uniform(0.01, 0.1)) # try to obfuscate delays and emulate network delays + # print(f'thread {message.source} is in queue') + if self.next_message and not message.source == self.pick_device: + self.collectThread() + return self.queue(message) + + self.parent.queue(self, message, True) + self.last_action = "send" + self.messages_sent.append(message) + + self.pick_running = False + + if self.is_stepping: + self.step() self._progress.release() - def _step(self, message:str = ""): - if not self._single: - print(f'\t{self._messages_sent}: {message}') - while self._stepping: #run while waiting for input - if self._single: #break while if the desired action is a single message - self._single = False - break + # the main function to stop execution + def step(self): + if self.is_stepping: + self.step_barrier.wait() - def on_press(self, key:keyboard.KeyCode): - try: - #for keycode class - key = key.char - except: - #for key class - key = key.name - if key == "f" or key == "enter": - self._stepping = False - elif key == "space" and not self._keyheld: - self._single = True - self._keyheld = True - - def on_release(self, key:keyboard.KeyCode): + def pick(self): + self.print_transit() + if self.parent is AsyncEmulator: + messages = self._messages + else: + messages = self._last_round_messages + keys = [] + for key in messages.keys(): + if len(messages[key]) > 0: + keys.append(key) + print(f"{GREEN}Available devices:{RESET} {keys}") + device = int(input("Specify device: ")) + self.print_transit_for_device(device) + index = int(input("Specify index of the next message: ")) + self.pick_device = device + self.next_message = messages[device][index] + while self.next_message: + self.pick_running = True + self.step_barrier.wait() + while self.pick_running and not self.all_terminated: + pass + sleep(0.1) + + def prompt(self): + self.prompt_active = True + line = "" + while not line == "exit": + sleep(1) + line = input( + f"\t[{CYAN}{len(self.messages_sent)} {RESET}->{CYAN} {len(self.messages_received)}{RESET}] > " + ) + self.input_lock.acquire() + args = line.split(" ") + match args[0]: + case "": + if not self.all_terminated: + self.step_barrier.wait() + case "queue": + if len(args) == 1: + self.print_transit() + else: + self.print_transit_for_device(int(args[1])) + case "exit": + self.is_stepping = False + self.step_barrier.wait() + case "swap": + self.swap_emulator() + case "pick": + try: + self.pick() + except ValueError: + pass + self.input_lock.release() + self.prompt_active = False + + def print_prompt(self): + print( + f"\t[{CYAN}{len(self.messages_sent)} {RESET}->{CYAN} {len(self.messages_received)}{RESET}] > ", + end="", + flush=True, + ) + + # print all messages in transit + def print_transit(self): + print(f"{CYAN}Messages in transit:{RESET}") + if self.parent is AsyncEmulator: + for messages in self._messages.values(): + for message in messages: + print(f"\t{message}") + elif self.parent is SyncEmulator: + for messages in self._last_round_messages.values(): + for message in messages: + print(f"\t{message}") + + # print all messages in transit to specified device + def print_transit_for_device(self, device): + print(f"{CYAN}Messages in transit to device #{device}{RESET}") + print(f"\t{CYAN}index{RESET}: ") + index = 0 + if self.parent is AsyncEmulator: + if device not in self._messages.keys(): + return + messages: list[MessageStub] = self._messages.get(device) + elif self.parent is SyncEmulator: + if device not in self._last_round_messages.keys(): + return + messages: list[MessageStub] = self._last_round_messages.get(device) + for message in messages: + print(f"\t{CYAN}{index}{RESET}: {message}") + index += 1 + + # swap between which parent class the program will run in between deliveries + def swap_emulator(self): + if self.parent is AsyncEmulator: + self.parent = SyncEmulator + elif self.parent is SyncEmulator: + self.parent = AsyncEmulator + print(f"Changed emulator to {GREEN}{self.parent.__name__}{RESET}") + + def run(self): + self._progress.acquire() + for index in self.ids: + self._awaits[index].acquire() + self._start_threads() + self._progress.release() + + self._round_lock.acquire() + while True: + if self.parent is AsyncEmulator: + sleep(0.1) + self._progress.acquire() + # check if everyone terminated + if self.all_terminated: + break + self._progress.release() + else: + self._round_lock.acquire() + # check if everyone terminated + self._progress.acquire() + print(f"\r\t## {GREEN}ROUND {self._rounds}{RESET} ##") + if self.all_terminated: + self._progress.release() + break + # send messages + for index in self.ids: + # intentionally change the order + if index in self._current_round_messages: + nxt = copy.deepcopy(self._current_round_messages[index]) + random.shuffle(nxt) + if index in self._last_round_messages: + self._last_round_messages[index] += nxt + else: + self._last_round_messages[index] = nxt + self._current_round_messages = {} + self.reset_done() + self._rounds += 1 + ids = [x for x in self.ids] # convert to list to make it shuffleable + random.shuffle(ids) + for index in ids: + if self._awaits[index].locked(): + self._awaits[index].release() + self._progress.release() + + for t in self._threads: + t.join() + + def done(self, id): + return self.parent.done(self, id) + + def _run_thread(self, index: int): + super()._run_thread(index) + self._devices[index]._finished = True + + def print_statistics(self): + return self.parent.print_statistics(self) + + def collectThread(self): + # print("collecting a thread") + self.pick_running = False + self._progress.release() try: - #for key class - key = key.char - except: - #for keycode class - key = key.name - if key == "f": - self._stepping = True - self._keyheld = False \ No newline at end of file + self.barrier.wait() + except BrokenBarrierError: + pass diff --git a/emulators/SyncEmulator.py b/emulators/SyncEmulator.py index 214412e..caf08b7 100644 --- a/emulators/SyncEmulator.py +++ b/emulators/SyncEmulator.py @@ -1,30 +1,39 @@ import copy import random import threading +from os import name from typing import Optional from emulators.EmulatorStub import EmulatorStub from emulators.MessageStub import MessageStub +if name == "posix": + RESET = "\u001B[0m" + CYAN = "\u001B[36m" + GREEN = "\u001B[32m" +else: + RESET = "" + CYAN = "" + GREEN = "" -class SyncEmulator(EmulatorStub): +class SyncEmulator(EmulatorStub): def __init__(self, number_of_devices: int, kind): super().__init__(number_of_devices, kind) self._round_lock = threading.Lock() - self._done = [False for _ in self.ids()] - self._awaits = [threading.Lock() for _ in self.ids()] - self._last_round_messages = {} + self._done = [False for _ in self.ids] + self._awaits = [threading.Lock() for _ in self.ids] + self._last_round_messages: dict[int, list[MessageStub]] = {} self._current_round_messages = {} self._messages_sent = 0 self._rounds = 0 def reset_done(self): - self._done = [False for _ in self.ids()] + self._done = [False for _ in self.ids] def run(self): self._progress.acquire() - for index in self.ids(): + for index in self.ids: self._awaits[index].acquire() self._start_threads() self._progress.release() @@ -35,12 +44,12 @@ def run(self): self._round_lock.acquire() # check if everyone terminated self._progress.acquire() - print(f'## ROUND {self._rounds} ##') - if self.all_terminated(): + print(f"\r\t## {GREEN}ROUND {self._rounds}{RESET} ##") + if self.all_terminated: self._progress.release() break # send messages - for index in self.ids(): + for index in self.ids: # intentionally change the order if index in self._current_round_messages: nxt = copy.deepcopy(self._current_round_messages[index]) @@ -52,7 +61,7 @@ def run(self): self._current_round_messages = {} self.reset_done() self._rounds += 1 - ids = [x for x in self.ids()] # convert to list to make it shuffleable + ids = [x for x in self.ids] # convert to list to make it shuffleable random.shuffle(ids) for index in ids: if self._awaits[index].locked(): @@ -62,27 +71,35 @@ def run(self): t.join() return - def queue(self, message: MessageStub): - self._progress.acquire() + def queue(self, message: MessageStub, stepper=False): + if not stepper: + self._progress.acquire() self._messages_sent += 1 - print(f'\tSend {message}') + print(f"\r\t{GREEN}Send{RESET} {message}") if message.destination not in self._current_round_messages: self._current_round_messages[message.destination] = [] - self._current_round_messages[message.destination].append(copy.deepcopy(message)) # avoid accidental memory sharing - self._progress.release() + self._current_round_messages[message.destination].append( + copy.deepcopy(message) + ) # avoid accidental memory sharing + if not stepper: + self._progress.release() - def dequeue(self, index: int) -> Optional[MessageStub]: - self._progress.acquire() + def dequeue(self, index: int, stepper=False) -> Optional[MessageStub]: + if not stepper: + self._progress.acquire() if index not in self._last_round_messages: - self._progress.release() + if not stepper: + self._progress.release() return None elif len(self._last_round_messages[index]) == 0: - self._progress.release() + if not stepper: + self._progress.release() return None else: m = self._last_round_messages[index].pop() - print(f'\tReceive {m}') - self._progress.release() + print(f"\r\t{GREEN}Receive{RESET} {m}") + if not stepper: + self._progress.release() return m def done(self, index: int): @@ -90,26 +107,28 @@ def done(self, index: int): if self._done[index]: # marked as done twice! self._progress.release() - raise RuntimeError(f'Device {index} called wait_for_next_round() twice in the same round!') + raise RuntimeError( + f"Device {index} called wait_for_next_round() twice in the same round!" + ) self._done[index] = True # check if the thread have marked their round as done OR have ended - if all([self._done[x] or not self._threads[x].is_alive() for x in self.ids()]): + if all([self._done[x] or not self._threads[x].is_alive() for x in self.ids]): self._round_lock.release() self._progress.release() self._awaits[index].acquire() - def print_statistics(self): - print(f'\tTotal {self._messages_sent} messages') - print(f'\tAverage {self._messages_sent/len(self._devices)} messages/device') - print(f'\tTotal {self._rounds} rounds') + print(f"\t{GREEN}Total:{RESET} {self._messages_sent} messages") + print( + f"\t{GREEN}Average:{RESET} {self._messages_sent/len(self._devices)} messages/device" + ) + print(f"\t{GREEN}Total:{RESET} {self._rounds} rounds") - def terminated(self, index:int): + def terminated(self, index: int): self._progress.acquire() self._done[index] = True - if all([self._done[x] or not self._threads[x].is_alive() - for x in self.ids()]): + if all([self._done[x] or not self._threads[x].is_alive() for x in self.ids]): if self._round_lock.locked(): self._round_lock.release() self._progress.release() diff --git a/emulators/exercise_overlay.py b/emulators/exercise_overlay.py new file mode 100644 index 0000000..163960e --- /dev/null +++ b/emulators/exercise_overlay.py @@ -0,0 +1,461 @@ +from threading import Thread +from time import sleep +from PyQt6.QtWidgets import ( + QWidget, + QApplication, + QHBoxLayout, + QVBoxLayout, + QPushButton, + QTabWidget, + QLabel, +) +from PyQt6.QtGui import QIcon +from PyQt6.QtCore import Qt +from sys import argv +from os import name +from math import cos, sin, pi +from emulators.AsyncEmulator import AsyncEmulator +from emulators.MessageStub import MessageStub + +from emulators.table import Table +from emulators.SteppingEmulator import SteppingEmulator + +if name == "posix": + RESET = "\u001B[0m" + CYAN = "\u001B[36m" + GREEN = "\u001B[32m" + RED = "\u001B[31m" +else: + RESET = "" + CYAN = "" + GREEN = "" + RED = "" + + +def circle_button_style(size, color="black"): + return f""" + QPushButton {{ + background-color: transparent; + border-style: solid; + border-width: 2px; + border-radius: {int(size / 2)}px; + border-color: {color}; + max-width: {size}px; + max-height: {size}px; + min-width: {size}px; + min-height: {size}px; + }} + QPushButton:hover {{ + background-color: gray; + border-width: 2px; + }} + QPushButton:pressed {{ + background-color: transparent; + border-width: 1px + }} + """ + + +class Window(QWidget): + h = 640 + w = 600 + device_size = 80 + last_message = None + windows = list() + + def __init__(self, elements, restart_function, emulator: SteppingEmulator): + super().__init__() + self.emulator = emulator + self.setFixedSize(self.w, self.h) + layout = QVBoxLayout() + tabs = QTabWidget() + tabs.setFixedSize(self.w - 20, self.h - 20) + self.buttons: dict[int, QPushButton] = {} + tabs.addTab(self.main(elements, restart_function), "Main") + tabs.addTab(self.controls(), "controls") + layout.addWidget(tabs) + self.setLayout(layout) + self.setWindowTitle("Stepping Emulator") + self.setWindowIcon(QIcon("icon.ico")) + self.set_device_color() + self.pick_window = False + self.queue_window = False + self.all_data_window = False + + def coordinates(self, center, r, i, n): + x = sin((i * 2 * pi) / n) + y = cos((i * 2 * pi) / n) + if x < pi: + return int(center[0] - (r * x)), int(center[1] - (r * y)) + else: + return int(center[0] - (r * -x)), int(center[1] - (r * y)) + + def show_device_data(self, device_id): + def show(): + received: list[MessageStub] = list() + sent: list[MessageStub] = list() + for message in self.emulator.messages_received: + if message.destination == device_id: + received.append(message) + if message.source == device_id: + sent.append(message) + if len(received) > len(sent): + for _ in range(len(received) - len(sent)): + sent.append("") + elif len(sent) > len(received): + for _ in range(len(sent) - len(received)): + received.append("") + content = list() + for i in range(len(received)): + if received[i] == "": + msg = ( + str(sent[i]) + .replace(f"{sent[i].source} -> {sent[i].destination} : ", "") + .replace(f"{sent[i].source}->{sent[i].destination} : ", "") + ) + content.append(["", received[i], str(sent[i].destination), msg]) + elif sent[i] == "": + msg = ( + str(received[i]) + .replace( + f"{received[i].source} -> {received[i].destination} : ", "" + ) + .replace( + f"{received[i].source}->{received[i].destination} : ", "" + ) + ) + content.append([str(received[i].source), msg, "", sent[i]]) + else: + sent_msg = ( + str(sent[i]) + .replace(f"{sent[i].source} -> {sent[i].destination} : ", "") + .replace(f"{sent[i].source}->{sent[i].destination} : ", "") + ) + received_msg = ( + str(received[i]) + .replace( + f"{received[i].source} -> {received[i].destination} : ", "" + ) + .replace( + f"{received[i].source}->{received[i].destination} : ", "" + ) + ) + content.append( + [ + str(received[i].source), + received_msg, + str(sent[i].destination), + sent_msg, + ] + ) + content.insert(0, ["Source", "Message", "Destination", "Message"]) + table = Table(content, title=f"Device #{device_id}") + self.windows.append(table) + table.setFixedSize(300, 500) + table.show() + return table + + return show + + def show_all_data(self): + if self.all_data_window: + return + self.all_data_window = True + content = [] + messages = self.emulator.messages_sent + message_content = [] + for message in messages: + temp = str(message) + temp = temp.replace(f"{message.source} -> {message.destination} : ", "") + temp = temp.replace(f"{message.source}->{message.destination} : ", "") + message_content.append(temp) + + content = [ + [ + str(messages[i].source), + str(messages[i].destination), + message_content[i], + str(i), + ] + for i in range(len(messages)) + ] + content.insert(0, ["Source", "Destination", "Message", "Sequence number"]) + parent = self + + class MyTable(Table): + def closeEvent(self, event): + parent.all_data_window = False + return super().closeEvent(event) + + table = MyTable(content, title="All data") + self.windows.append(table) + table.setFixedSize(500, 500) + table.show() + return table + + def show_queue(self): + if self.queue_window: + return + self.queue_window = True + content = [["Source", "Destination", "Message"]] + if self.emulator.parent is AsyncEmulator: + queue = self.emulator._messages.values() + else: + queue = self.emulator._last_round_messages.values() + for messages in queue: + for message in messages: + message_stripped = ( + str(message) + .replace(f"{message.source} -> {message.destination} : ", "") + .replace(f"{message.source}->{message.destination} : ", "") + ) + content.append( + [str(message.source), str(message.destination), message_stripped] + ) + parent = self + + class MyWidget(QWidget): + def closeEvent(self, event): + parent.queue_window = False + return super().closeEvent(event) + + window = MyWidget() + layout = QVBoxLayout() + table = Table(content, "Message queue") + layout.addWidget(table) + window.setLayout(layout) + self.windows.append(window) + window.setFixedSize(500, 500) + window.show() + + def pick(self): + if self.pick_window: + return + self.pick_window = True + + def execute(device, index): + def inner_execute(): + if self.emulator._devices[device]._finished: + table.destroy(True, True) + print( + f"{RED}The selected device has already finished execution!{RESET}" + ) + return + if self.emulator.parent is AsyncEmulator: + message = self.emulator._messages[device][index] + else: + message = self.emulator._last_round_messages[device][index] + + print(f"\r{CYAN}Choice from pick command{RESET}: {message}") + + self.emulator.pick_device = device + self.emulator.next_message = message + table.destroy(True, True) + self.pick_window = False + size = len(self.emulator.messages_received) + while self.emulator.next_message: + self.emulator.pick_running = True + self.step() + while ( + self.emulator.pick_running and not self.emulator.all_terminated + ): + pass + sleep(0.1) + + assert len(self.emulator.messages_received) == size + 1 + + return inner_execute + + keys = [] + if self.emulator.parent is AsyncEmulator: + messages = self.emulator._messages + else: + messages = self.emulator._last_round_messages + for item in messages.items(): + keys.append(item[0]) + keys.sort() + max_size = 0 + for m in messages.values(): + if len(m) > max_size: + max_size = len(m) + + content = [] + for i in range(max_size): + content.append([]) + for key in keys: + if len(messages[key]) > i: + button = QPushButton(str(messages[key][i])) + function_reference = execute(key, i) + button.clicked.connect(function_reference) + content[i].append(button) + else: + content[i].append("") + content.insert(0, [f"Device {key}" for key in keys]) + content[0].insert(0, "Message #") + for i in range(max_size): + content[i + 1].insert(0, str(i)) + + parent = self + + class MyTable(Table): + def closeEvent(self, event): + parent.pick_window = False + return super().closeEvent(event) + + table = MyTable(content, "Pick a message to be transmitted next to a device") + table.setFixedSize(150 * len(self.emulator._devices) + 1, 400) + table.show() + + def end(self): + if self.emulator.all_terminated: + return + self.emulator.is_stepping = False + self.emulator.step_barrier.wait() + while not self.emulator.all_terminated: + self.set_device_color() + sleep(0.1) + # self.emulator.print_prompt() + + def set_device_color(self): + sleep(0.1) + messages = ( + self.emulator.messages_sent + if self.emulator.last_action == "send" + else self.emulator.messages_received + ) + if len(messages) != 0: + last_message = messages[len(messages) - 1] + if not last_message == self.last_message: + for button in self.buttons.values(): + button.setStyleSheet(circle_button_style(self.device_size)) + if last_message.source == last_message.destination: + self.buttons[last_message.source].setStyleSheet( + circle_button_style(self.device_size, "yellow") + ) + else: + self.buttons[last_message.source].setStyleSheet( + circle_button_style(self.device_size, "green") + ) + self.buttons[last_message.destination].setStyleSheet( + circle_button_style(self.device_size, "red") + ) + self.last_message = last_message + + def step(self): + self.emulator.input_lock.acquire() + if not self.emulator.all_terminated: + self.emulator.step_barrier.wait() + self.emulator.input_lock.release() + + while self.emulator.step_barrier.n_waiting == 0: + pass + self.set_device_color() + # self.emulator.print_prompt() + + def restart_algorithm(self, function): + # if self.emulator.prompt_active: + # print(f'\r{RED}Please type "exit" in the prompt below{RESET}') + # self.emulator.print_prompt() + # return + self.windows.append(function()) + + def main(self, num_devices, restart_function): + main_tab = QWidget() + green = QLabel("green: source", main_tab) + green.setStyleSheet("color: green;") + green.move(5, 0) + green.show() + red = QLabel("red: destination", main_tab) + red.setStyleSheet("color: red;") + red.move(5, 20) + red.show() + yellow = QLabel("yellow: same device", main_tab) + yellow.setStyleSheet("color: yellow;") + yellow.move(5, 40) + yellow.show() + layout = QVBoxLayout() + device_area = QWidget() + device_area.setFixedSize(500, 500) + layout.addWidget(device_area) + main_tab.setLayout(layout) + for i in range(num_devices): + x, y = self.coordinates( + (device_area.width() / 2, device_area.height() / 2), + (device_area.height() / 2) - (self.device_size / 2), + i, + num_devices, + ) + button = QPushButton(f"Device #{i}", main_tab) + button.resize(self.device_size, self.device_size) + button.setStyleSheet(circle_button_style(self.device_size)) + button.move(x, int(y - (self.device_size / 2))) + button.clicked.connect(self.show_device_data(i)) + self.buttons[i] = button + + button_actions = { + "Step": self.step, + "End": self.end, + "Restart algorithm": lambda: self.restart_algorithm(restart_function), + "Show all messages": self.show_all_data, + "Switch emulator": self.swap_emulator, + "Show queue": self.show_queue, + "Pick": self.pick, + } + inner_layout = QHBoxLayout() + index = 0 + for action in button_actions.items(): + index += 1 + if index == 4: + layout.addLayout(inner_layout) + inner_layout = QHBoxLayout() + button = QPushButton(action[0]) + button.clicked.connect(action[1]) + inner_layout.addWidget(button) + layout.addLayout(inner_layout) + + return main_tab + + def swap_emulator(self): + self.emulator.input_lock.acquire() + print() + self.emulator.swap_emulator() + # self.emulator.print_prompt() + self.emulator.input_lock.release() + + def controls(self): + controls_tab = QWidget() + content = { + "step(press return)": "Step a single time through messages", + "exit": "Finish the execution of the algorithm", + "queue": "Show all messages currently waiting to be transmitted", + "queue ": "Show all messages currently waiting to be transmitted to a specific device", + "pick": "Pick the next message waiting to be transmitted to transmit next", + "swap": "Toggle between sync and async emulation", + } + main = QVBoxLayout() + main.setAlignment(Qt.AlignmentFlag.AlignTop) + controls = QLabel("Controls") + controls.setAlignment(Qt.AlignmentFlag.AlignCenter) + main.addWidget(controls) + top = QHBoxLayout() + key_layout = QVBoxLayout() + value_layout = QVBoxLayout() + for key in content.keys(): + key_layout.addWidget(QLabel(key)) + value_layout.addWidget(QLabel(content[key])) + top.addLayout(key_layout) + top.addLayout(value_layout) + main.addLayout(top) + controls_tab.setLayout(main) + return controls_tab + + def closeEvent(self, event): + Thread(target=self.end).start() + event.accept() + + +if __name__ == "__main__": + app = QApplication(argv) + window = Window(argv[1] if len(argv) > 1 else 10, lambda: print("Restart function")) + + app.exec() diff --git a/emulators/table.py b/emulators/table.py new file mode 100644 index 0000000..460073b --- /dev/null +++ b/emulators/table.py @@ -0,0 +1,42 @@ +from PyQt6.QtWidgets import ( + QWidget, + QHBoxLayout, + QVBoxLayout, + QLabel, + QScrollArea, +) +from PyQt6.QtGui import QIcon +from PyQt6.QtCore import Qt + + +class Table(QScrollArea): + def __init__(self, content: list[list[str | QWidget]], title="table"): + super().__init__() + widget = QWidget() + self.setWindowIcon(QIcon("icon.ico")) + self.setWindowTitle(title) + columns = QVBoxLayout() + columns.setAlignment(Qt.AlignmentFlag.AlignTop) + for row in content: + column = QHBoxLayout() + for element in row: + if isinstance(element, str): + label = QLabel(element) + label.setAlignment(Qt.AlignmentFlag.AlignCenter) + else: + label = element + column.addWidget(label) + columns.addLayout(column) + widget.setLayout(columns) + self.setWidgetResizable(True) + self.setWidget(widget) + + +if __name__ == "__main__": + from PyQt6.QtWidgets import QApplication + from sys import argv + + app = QApplication(argv) + table = Table([[str(i + j) for i in range(5)] for j in range(5)]) + table.show() + app.exec() diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..b650447 --- /dev/null +++ b/environment.yml @@ -0,0 +1,7 @@ +name: ds +dependencies: + - cryptography >= 37.0.4 + - pip + - pip: + - PyQt6 >= 6.3.1 + - pynput >= 1.7.6 \ No newline at end of file diff --git a/exercise_runner.py b/exercise_runner.py index 2b09350..0998bf2 100644 --- a/exercise_runner.py +++ b/exercise_runner.py @@ -1,76 +1,147 @@ import argparse +import importlib import inspect +from os import name +from threading import Thread -import exercises.exercise1 -import exercises.exercise2 -import exercises.exercise4 -import exercises.exercise5 -import exercises.exercise6 -import exercises.exercise7 -import exercises.exercise8 -import exercises.exercise9 -import exercises.exercise10 -import exercises.exercise11 -import exercises.exercise12 -import exercises.demo +from emulators.exercise_overlay import Window from emulators.AsyncEmulator import AsyncEmulator from emulators.SyncEmulator import SyncEmulator from emulators.SteppingEmulator import SteppingEmulator +if name == "posix": + RESET = "\u001B[0m" + CYAN = "\u001B[36m" + GREEN = "\u001B[32m" +else: + RESET = "" + CYAN = "" + GREEN = "" + + def fetch_alg(lecture: str, algorithm: str): - if '.' in algorithm or ';' in algorithm: - raise ValueError(f'"." and ";" are not allowed as names of solutions.') + if "." in algorithm or ";" in algorithm: + raise ValueError('"." and ";" are not allowed as names of solutions.') try: - alg = eval(f'exercises.{lecture}.{algorithm}') + module = importlib.import_module(f"exercises.{lecture}") + alg = getattr(module, algorithm) if not inspect.isclass(alg): raise TypeError(f'Could not find "exercises.{lecture}.{algorithm} class') - except: + except NameError: raise TypeError(f'Could not find "exercises.{lecture}.{algorithm} class') return alg -def run_exercise(lecture_no: int, algorithm: str, network_type: str, number_of_devices: int): +def run_exercise( + lecture_no: int, + algorithm: str, + network_type: str, + number_of_devices: int, + gui: bool, +): print( - f'Running Lecture {lecture_no} Algorithm {algorithm} in a network of type [{network_type}] using {number_of_devices} devices') + f"Running Lecture {lecture_no} Algorithm {algorithm} in a network of type [{network_type}] using {number_of_devices} devices" + ) if number_of_devices < 2: - raise IndexError(f'At least two devices are needed as an input argument, got {number_of_devices}') + raise IndexError( + f"At least two devices are needed as an input argument, got {number_of_devices}" + ) emulator = None - if network_type == 'async': + if network_type == "async": emulator = AsyncEmulator - elif network_type == 'sync': + elif network_type == "sync": emulator = SyncEmulator - elif network_type == 'stepping': + elif network_type == "stepping": emulator = SteppingEmulator instance = None if lecture_no == 0: - alg = fetch_alg('demo', 'PingPong') + alg = fetch_alg("demo", "PingPong") instance = emulator(number_of_devices, alg) else: - alg = fetch_alg(f'exercise{lecture_no}', algorithm) + alg = fetch_alg(f"exercise{lecture_no}", algorithm) instance = emulator(number_of_devices, alg) - if instance is not None: - instance.run() - print(f'Execution Complete') - instance.print_result() - print('Statistics') - instance.print_statistics() - else: - raise NotImplementedError( - f'You are trying to run an exercise ({algorithm}) of a lecture ({lecture_no}) which has not yet been released') + def run_instance(): + if instance: + instance.run() + print(f"{CYAN}Execution Complete{RESET}") + instance.print_result() + print(f"{CYAN}Statistics{RESET}") + instance.print_statistics() + else: + raise NotImplementedError( + f"You are trying to run an exercise ({algorithm}) of a lecture ({lecture_no}) which has not yet been released" + ) + + Thread(target=run_instance).start() + if isinstance(instance, SteppingEmulator) and gui: + window = Window( + number_of_devices, + lambda: run_exercise( + lecture_no, algorithm, network_type, number_of_devices, True + ), + instance, + ) + window.show() + return window + if not gui and isinstance(instance, SteppingEmulator): + instance.shell() if __name__ == "__main__": - parser = argparse.ArgumentParser(description='For exercises in Distributed Systems.') - parser.add_argument('--lecture', metavar='N', type=int, nargs=1, - help='Lecture number', required=True, choices=[0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12]) - parser.add_argument('--algorithm', metavar='alg', type=str, nargs=1, - help='Which algorithm from the exercise to run', required=True) - parser.add_argument('--type', metavar='nw', type=str, nargs=1, - help='whether to use [async] or [sync] network', required=True, choices=['async', 'sync', 'stepping']) - parser.add_argument('--devices', metavar='N', type=int, nargs=1, - help='Number of devices to run', required=True) + parser = argparse.ArgumentParser( + description="For exercises in Distributed Systems." + ) + parser.add_argument( + "--lecture", + metavar="N", + type=int, + nargs=1, + help="Lecture number", + required=True, + choices=[0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12], + ) + parser.add_argument( + "--algorithm", + metavar="alg", + type=str, + nargs=1, + help="Which algorithm from the exercise to run", + required=True, + ) + parser.add_argument( + "--type", + metavar="nw", + type=str, + nargs=1, + help="whether to use [async] or [sync] network", + required=True, + choices=["async", "sync", "stepping"], + ) + parser.add_argument( + "--devices", + metavar="N", + type=int, + nargs=1, + help="Number of devices to run", + required=True, + ) + parser.add_argument( + "--gui", action="store_true", help="Toggle the gui or cli", required=False + ) args = parser.parse_args() + import sys + + if args.gui and args.type[0] == "stepping": + from PyQt6.QtWidgets import QApplication - run_exercise(args.lecture[0], args.algorithm[0], args.type[0], args.devices[0]) \ No newline at end of file + app = QApplication(sys.argv) + run_exercise( + args.lecture[0], args.algorithm[0], args.type[0], args.devices[0], True + ) + app.exec() + else: + run_exercise( + args.lecture[0], args.algorithm[0], args.type[0], args.devices[0], False + ) diff --git a/exercise_runner_overlay.py b/exercise_runner_overlay.py new file mode 100644 index 0000000..5a5b3e8 --- /dev/null +++ b/exercise_runner_overlay.py @@ -0,0 +1,125 @@ +import typing +from PyQt6 import QtCore +from exercise_runner import run_exercise +from PyQt6.QtWidgets import * +from PyQt6.QtGui import QIcon, QRegularExpressionValidator +from PyQt6.QtCore import Qt, QThread, QObject, pyqtSignal, QRegularExpression +from sys import argv + +class Worker(QObject): + finished: pyqtSignal = pyqtSignal() + window: pyqtSignal = pyqtSignal(QObject) + def __init__(self) -> None: + super().__init__() + + def run(self, + lecture_no: int, + algorithm: str, + network_type: str, + number_of_devices: int, + ): + self.window.emit(run_exercise(lecture_no, algorithm, network_type, number_of_devices, True)) + self.finished.emit() + + +class InputArea(QFormLayout): + def __init__(self) -> None: + super().__init__() + self.__algs = [ + "PingPong", + "Gossip", + "RipCommunication", + "TokenRing", + "TOSEQMulticast", + "PAXOS", + "Bully", + "GfsNetwork", + "MapReduceNetwork", + "BlockchainNetwork", + "ChordNetwork", + "AodvNode", + ] + self.lecture = QComboBox() + self.lecture.addItems([str(i) for i in range(13) if i != 3]) + + self.type = QComboBox() + self.type.addItems(["stepping", "async", "sync"]) + + self.alg = QComboBox() + self.alg.addItems(self.__algs) + + self.device = QLineEdit(text="3") + + self.device.setValidator( + QRegularExpressionValidator( + QRegularExpression(r"[0-9]*") + )) + + self.addRow("Lecture", self.lecture) + self.addRow("Type", self.type) + self.addRow("Algorithm", self.alg) + self.addRow("Devices", self.device) + + self.lecture.currentTextChanged.connect(self.__lecture_handler) + self.__lecture_handler("0") + + def __lecture_handler(self, text: str): + self.alg.setCurrentText(self.__algs[int(text)]) + + def data(self): + return ( + int(self.lecture.currentText()), + self.alg.currentText(), + self.type.currentText(), + int(self.device.text()) + ) + +class MainWindow(QMainWindow): + def __init__(self) -> None: + super().__init__( + windowTitle="Distributed Exercises AAU", + windowIcon=QIcon("icon.ico") + ) + self.__thread = None + self.__worker = None + self.__windows = [] + layout = QVBoxLayout() + self.__start_btn = QPushButton("Start") + + group = QGroupBox(title="Inputs") + self.__input_area = InputArea() + group.setLayout(self.__input_area) + + layout.addWidget(group) + layout.addWidget(self.__start_btn) + widget = QWidget() + widget.setLayout(layout) + self.setCentralWidget(widget) + + self.__start_btn.clicked.connect(self.__start) + + def __start(self): + if self.__worker is not None: + return + + self.__thread = QThread() + self.__worker = Worker() + self.__thread.started.connect(lambda: self.__worker.run(*self.__input_area.data())) + self.__worker.finished.connect(self.__thread.quit) + self.__worker.window.connect(self.__window_handler) + self.__worker.finished.connect(self.__worker.deleteLater) + self.__thread.finished.connect(self.__thread.deleteLater) + self.__thread.start() + + def __window_handler(self, window: QObject): + self.__windows.append(window) + + def show(self) -> None: + self.resize(600, 100) + return super().show() + + +app = QApplication(argv) +window = MainWindow() +window.show() +app.exec() diff --git a/exercises/demo.py b/exercises/demo.py index ef3fc24..03a5109 100644 --- a/exercises/demo.py +++ b/exercises/demo.py @@ -7,7 +7,6 @@ # We extend the MessageStub here for the message-types we wish to communicate class PingMessage(MessageStub): - # the constructor-function takes the source and destination as arguments. These are used for "routing" but also # for pretty-printing. Here we also take the specific flag of "is_ping" def __init__(self, sender: int, destination: int, is_ping: bool): @@ -18,12 +17,14 @@ def __init__(self, sender: int, destination: int, is_ping: bool): # remember to implement the __str__ method such that the debug of the framework works! def __str__(self): - return f'{self.source} -> {self.destination} : ping? {self.is_ping}' + if self.is_ping: + return f"{self.source} -> {self.destination} : Ping" + else: + return f"{self.source} -> {self.destination} : Pong" # This class extends on the basic Device class. We will implement the protocol in the run method class PingPong(Device): - # The constructor must have exactly this form. def __init__(self, index: int, number_of_devices: int, medium: Medium): # forward the constructor arguments to the super-constructor @@ -38,13 +39,17 @@ def run(self): # for this algorithm, we will repeat the protocol 10 times and then stop for repetetions in range(0, 10): # in each repetition, let us send the ping to one random other device - message = PingMessage(self.index(), random.randrange(0, self.number_of_devices()), self._is_ping) + message = PingMessage( + self.index, + random.randrange(0, self.number_of_devices), + self._is_ping, + ) # we send the message via a "medium" - self.medium().send(message) + self.medium.send(message) # in this instance, we also try to receive some messages, there can be multiple, but # eventually the medium will return "None" while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is None: break @@ -62,8 +67,10 @@ def run(self): self._is_ping = ingoing.is_ping # this call is only used for synchronous networks - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() # for pretty-printing and debugging, implement this function def print_result(self): - print(f'\tDevice {self.index()} got pings: {self._rec_ping} and pongs: {self._rec_pong}') + print( + f"\tDevice {self.index} got pings: {self._rec_ping} and pongs: {self._rec_pong}" + ) diff --git a/exercises/exercise1.py b/exercises/exercise1.py index 32b83ba..de8b928 100644 --- a/exercises/exercise1.py +++ b/exercises/exercise1.py @@ -4,18 +4,16 @@ class GossipMessage(MessageStub): - def __init__(self, sender: int, destination: int, secrets): super().__init__(sender, destination) # we use a set to keep the "secrets" here self.secrets = secrets def __str__(self): - return f'{self.source} -> {self.destination} : {self.secrets}' + return f"{self.source} -> {self.destination} : {self.secrets}" class Gossip(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) # for this exercise we use the index as the "secret", but it could have been a new routing-table (for instance) @@ -29,4 +27,4 @@ def run(self): return def print_result(self): - print(f'\tDevice {self.index()} got secrets: {self._secrets}') + print(f"\tDevice {self.index} got secrets: {self._secrets}") diff --git a/exercises/exercise10.py b/exercises/exercise10.py index eeda387..bcbc949 100644 --- a/exercises/exercise10.py +++ b/exercises/exercise10.py @@ -1,21 +1,12 @@ -import math -import random -import sys -import os - from emulators.Device import Device from emulators.Medium import Medium from emulators.MessageStub import MessageStub -from cryptography.hazmat.primitives import hashes from hashlib import sha256 import json import time - - - class Block: def __init__(self, index, transactions, timestamp, previous_hash, nonce=0): self.index = index @@ -37,10 +28,8 @@ def hash_binary(self): return "{0:0256b}".format(int(self.hash, 16)) - -class Blockchain(): +class Blockchain: def __init__(self): - super().__init__() self.unconfirmed_transactions = [] self.chain = [] @@ -58,11 +47,12 @@ def last_block(self): return self.chain[-1] difficulty = 4 + # TODO: set difficulty to 2, to have many more forks. # TODO: understand why having lower difficulty leads to more forks. def proof_of_work(self, block): computed_hash_binary = block.hash_binary - if not computed_hash_binary.startswith('0' * Blockchain.difficulty): + if not computed_hash_binary.startswith("0" * Blockchain.difficulty): return False return True @@ -77,7 +67,7 @@ def add_block(self, block): return False self.chain.append(block) return True - + def add_new_transaction(self, transaction): self.unconfirmed_transactions.append(transaction) @@ -88,7 +78,6 @@ def to_string(self): return msg - class BlockchainMiner(Device): def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) @@ -101,11 +90,13 @@ def try_mining(self): last_block = self.blockchain.last_block # I create a block using current timestamp, unconfirmed transactions and nonce - new_block = Block(index=last_block.index + 1, - transactions=self.blockchain.unconfirmed_transactions, - timestamp=time.time(), - previous_hash=last_block.hash, - nonce=self.next_nonce) + new_block = Block( + index=last_block.index + 1, + transactions=self.blockchain.unconfirmed_transactions, + timestamp=time.time(), + previous_hash=last_block.hash, + nonce=self.next_nonce, + ) # I check if the block passes the proof_of_work test proof = self.blockchain.proof_of_work(new_block) @@ -126,17 +117,17 @@ def try_mining(self): def disseminate_chain(self): # I send the blockchain to everybody for m in BlockchainNetwork.miners: - if not m == self.index(): - message = BlockchainMessage(self.index(), m, self.blockchain.chain) - self.medium().send(message) + if not m == self.index: + message = BlockchainMessage(self.index, m, self.blockchain.chain) + self.medium.send(message) # Since I flushed the unconfirmed transactions, I assign the incentives to myself for next block, in case I will be the winner of the proof of work test - self.blockchain.add_new_transaction(f"(miner {self.index()} gets incentive)") + self.blockchain.add_new_transaction(f"(miner {self.index} gets incentive)") def do_some_work(self): # if the chain is empty and index == 0, create the genesis block and disseminate the blockchain # if index is not 0, do nothing if len(self.blockchain.chain) == 0: - if self.index() == 0: + if self.index == 0: self.blockchain.create_genesis_block() self.disseminate_chain() else: @@ -147,14 +138,14 @@ def do_some_work(self): def run(self): # I assign the incentives to myself, in case I add the next block - self.blockchain.add_new_transaction(f"(miner {self.index()} gets incentive)") + self.blockchain.add_new_transaction(f"(miner {self.index} gets incentive)") # since this is a miner, it tries to mine, then it looks for incoming requests (messages) to accumulate transactions etc while True: self.do_some_work() - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, BlockchainMessage): @@ -164,19 +155,24 @@ def handle_ingoing(self, ingoing: MessageStub): pass elif isinstance(ingoing, BlockchainRequestMessage): # this is used to send the blockchain data to a client requesting them - message = BlockchainMessage(self.index(), ingoing.source, self.blockchain.chain) - self.medium().send(message) + message = BlockchainMessage( + self.index, ingoing.source, self.blockchain.chain + ) + self.medium.send(message) + elif isinstance(ingoing, TransactionMessage): + self.blockchain.add_new_transaction(ingoing.transaction) elif isinstance(ingoing, QuitMessage): return False return True def print_result(self): - print("Miner " + str(self.index()) + " quits") - + print("Miner " + str(self.index) + " quits") class BlockchainClient(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium, my_miner: int): + def __init__( + self, index: int, number_of_devices: int, medium: Medium, my_miner: int + ): super().__init__(index, number_of_devices, medium) self.my_miner = my_miner @@ -184,18 +180,20 @@ def run(self): # the client spends its time adding transactions (reasonable) and asking how long the blockchain is (unreasonable, but used for the termination) self.request_blockchain() while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def send_transaction(self): - message = TransactionMessage(self.index(), self.my_miner, f"(transaction by client {self.index()})") - self.medium().send(message) + message = TransactionMessage( + self.index, self.my_miner, f"(transaction by client {self.index})" + ) + self.medium.send(message) def request_blockchain(self): - message = BlockchainRequestMessage(self.index(), self.my_miner) - self.medium().send(message) + message = BlockchainRequestMessage(self.index, self.my_miner) + self.medium.send(message) def handle_ingoing(self, ingoing: MessageStub): # the termination clause is *very* random @@ -204,7 +202,7 @@ def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, BlockchainMessage): if target_len <= len(ingoing.chain): - self.medium().send(QuitMessage(self.index(), self.my_miner)) + self.medium.send(QuitMessage(self.index, self.my_miner)) return False # if I don't decide to quit, I add a transaction and request the blockchain data one more time self.send_transaction() @@ -212,12 +210,12 @@ def handle_ingoing(self, ingoing: MessageStub): return True def print_result(self): - print(f"client {self.index()} quits") - + print(f"client {self.index} quits") class BlockchainNetwork: miners = [] + def __new__(cls, index: int, number_of_devices: int, medium: Medium): # first miner MUST have index 0 @@ -225,8 +223,7 @@ def __new__(cls, index: int, number_of_devices: int, medium: Medium): return BlockchainMiner(index, number_of_devices, medium) else: # I associate a miner to each client, in a very aleatory manner - return BlockchainClient(index, number_of_devices, medium, index-1) - + return BlockchainClient(index, number_of_devices, medium, index - 1) class QuitMessage(MessageStub): @@ -234,8 +231,7 @@ def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'QUIT REQUEST {self.source} -> {self.destination}' - + return f"QUIT REQUEST {self.source} -> {self.destination}" class BlockchainMessage(MessageStub): @@ -244,8 +240,7 @@ def __init__(self, sender: int, destination: int, chain: list): self.chain = chain def __str__(self): - return f'NEW BLOCK MESSAGE {self.source} -> {self.destination}: ({len(self.chain)} blocks)' - + return f"NEW BLOCK MESSAGE {self.source} -> {self.destination}: ({len(self.chain)} blocks)" class TransactionMessage(MessageStub): @@ -254,8 +249,7 @@ def __init__(self, sender: int, destination: int, transaction: str): self.transaction = transaction def __str__(self): - return f'TRANSACTION {self.source} -> {self.destination}: ({self.transaction})' - + return f"TRANSACTION {self.source} -> {self.destination}: ({self.transaction})" class BlockchainRequestMessage(MessageStub): @@ -263,4 +257,4 @@ def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'REQUEST FOR BLOCKCHAIN DATA {self.source} -> {self.destination}' + return f"REQUEST FOR BLOCKCHAIN DATA {self.source} -> {self.destination}" diff --git a/exercises/exercise11.py b/exercises/exercise11.py index 2ffe227..d738375 100644 --- a/exercises/exercise11.py +++ b/exercises/exercise11.py @@ -1,12 +1,10 @@ -import math import random -import sys +from typing import Optional from emulators.Device import Device from emulators.Medium import Medium from emulators.MessageStub import MessageStub -import json import time @@ -17,22 +15,30 @@ # size for the chord addresses in bits address_size = 6 # only for initialization: -all_nodes = [] -all_routing_data = [] +all_nodes: list[int] = [] +all_routing_data: list["RoutingData"] = [] + + class RoutingData: # all tuples ("prev" and the ones in the finger_table) are (index, chord_id) - def __init__(self, index: int, chord_id: int, prev: tuple, finger_table: list): + def __init__( + self, + index: int, + chord_id: int, + prev: tuple[int, int], + finger_table: list[tuple[int, int]], + ): self.index = index self.chord_id = chord_id self.prev = prev self.finger_table = finger_table def to_string(self): - ret = f"node ({self.index}, {self.chord_id}) prev {self.prev} finger_table {self.finger_table}" + ret = f"Node ({self.index}, {self.chord_id}) prev {self.prev} finger_table {self.finger_table}" return ret -def in_between(candidate, low, high): +def in_between(candidate: int, low: int, high: int): # the function returns False when candidate == low or candidate == high # take care of those cases in the calling function if low == high: @@ -44,25 +50,32 @@ def in_between(candidate, low, high): class ChordNode(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium, connected: bool, routing_data: RoutingData): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + connected: bool, + routing_data: Optional[RoutingData], + ): super().__init__(index, number_of_devices, medium) self.connected = connected self.routing_data = routing_data - self.saved_data = [] + self.saved_data: list[str] = [] def run(self): # a chord node acts like a server while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() - def is_request_for_me(self, guid): + def is_request_for_me(self, guid: int) -> bool: # TODO: implement this function that checks if the routing process is over pass - def next_hop(self, guid): + def next_hop(self, guid: int) -> int: # TODO: implement this function with the routing logic pass @@ -75,8 +88,8 @@ def handle_ingoing(self, ingoing: MessageStub): # TODO: route the message # you can fill up the next_hop function for this next_hop = self.next_hop(ingoing.guid) - message = PutMessage(self.index(), next_hop[0], ingoing.guid, ingoing.data) - self.medium().send(message) + message = PutMessage(self.index, next_hop, ingoing.guid, ingoing.data) + self.medium.send(message) if isinstance(ingoing, GetReqMessage): # maybe TODO, but the GET is not very interesting pass @@ -104,10 +117,11 @@ def print_result(self): my_range = self.routing_data.chord_id - self.routing_data.prev[1] if my_range < 0: my_range += pow(2, address_size) - print(f"Chord node {self.index()} quits, it managed {my_range} addresses, it had {len(self.saved_data)} data blocks") + print( + f"Chord node {self.index} quits, it managed {my_range} addresses, it had {len(self.saved_data)} data blocks" + ) else: - print(f"Chord node {self.index()} quits, it was still disconnected") - + print(f"Chord node {self.index} quits, it was still disconnected") class ChordClient(Device): @@ -116,32 +130,34 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): def run(self): for i in range(pow(2, address_size)): - guid = i # I send a message to each address, to check if each node stores one data for each address it manages + guid = i + # I send a message to each address, to check if each node stores one data for each address it manages # if your chord address space gets too big, use the following code: # for i in range(pow(2, address_size)): # guid = random.randint(0, pow(2,address_size)-1) - message = PutMessage(self.index(), 2, guid, "hello") - self.medium().send(message) + message = PutMessage(self.index, 2, guid, "hello") + self.medium.send(message) # TODO: uncomment this code to start the JOIN process - #new_chord_id = random.randint(0, pow(2,address_size)-1) - #while new_chord_id in all_nodes: + # new_chord_id = random.randint(0, pow(2,address_size)-1) + # while new_chord_id in all_nodes: # new_chord_id = random.randint(0, pow(2,address_size)-1) - #message = StartJoinMessage(self.index(), 1, new_chord_id) - #self.medium().send(message) - - time.sleep(10) # or use some smart trick to wait for the routing process to be completed before shutting down the distributed system - for i in range(1, self.number_of_devices()): - message = QuitMessage(self.index(), i) - self.medium().send(message) + # message = StartJoinMessage(self.index(), 1, new_chord_id) + # self.medium().send(message) + + time.sleep(10) + # or use some smart trick to wait for the routing process to be completed before shutting down the distributed system + for i in range(1, self.number_of_devices): + message = QuitMessage(self.index, i) + self.medium.send(message) return - # currently, I do not manage incoming messages - while True: - for ingoing in self.medium().receive_all(): - if not self.handle_ingoing(ingoing): - return - self.medium().wait_for_next_round() + # currently, I do not manage incoming messages since there are no incoming messages for the client. + # while True: + # for ingoing in self.medium().receive_all(): + # if not self.handle_ingoing(ingoing): + # return + # self.medium().wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, QuitMessage): @@ -149,35 +165,55 @@ def handle_ingoing(self, ingoing: MessageStub): return True def print_result(self): - print(f"client {self.index()} quits") + print(f"client {self.index} quits") class ChordNetwork: - def init_routing_tables(number_of_devices): - N = number_of_devices-2 # routing_data 0 will be for device 2, etc + # Initializes routing tables for all nodes, except for the client and disconnected node + def init_routing_tables(number_of_devices: int): + # The first node is always the client, and the second is a disconnected node + # Therefore routing_data 0 will be for device 2, etc + N = number_of_devices - 2 + + # Populate the list of Chord IDs for the nodes while len(all_nodes) < N: - new_chord_id = random.randint(0, pow(2,address_size)-1) - if not new_chord_id in all_nodes: + new_chord_id = random.randint(0, pow(2, address_size) - 1) + if new_chord_id not in all_nodes: all_nodes.append(new_chord_id) + + # Sort to determine the correct positions within the Chord ring all_nodes.sort() + # Initialize routing data for each node, including predecessor and finger table for id in range(N): - prev_id = id-1 - if prev_id < 0: - prev_id += N - prev = (prev_id, all_nodes[prev_id]) + # Determine the previous node in the ring, using modulo for wrap-around + prev_id = (id - 1) % N + # Add 2 to get "message-able" device index + prev = (prev_id + 2, all_nodes[prev_id]) + + # Populate the finger table for each node new_finger_table = [] for i in range(address_size): + # Calculate the target ID for the current finger entry at_least = (all_nodes[id] + pow(2, i)) % pow(2, address_size) - candidate = (id+1)%N + candidate = (id + 1) % N + + # Find the appropriate node that should be the entry for this finger while in_between(all_nodes[candidate], all_nodes[id], at_least): - candidate = (candidate+1)%N - new_finger_table.append((candidate+2, all_nodes[candidate])) # I added 2 to candidate since routing_data 0 is for device 2, and so on - all_routing_data.append(RoutingData(id+2, all_nodes[id], prev, new_finger_table)) - print(RoutingData(id+2, all_nodes[id], prev, new_finger_table).to_string()) + candidate = (candidate + 1) % N + # I added 2 to candidate since routing_data 0 is for device 2, and so on + new_finger_table.append((candidate + 2, all_nodes[candidate])) + + # Store the routing data for the current node + all_routing_data.append( + RoutingData(id + 2, all_nodes[id], prev, new_finger_table) + ) + + print( + RoutingData(id + 2, all_nodes[id], prev, new_finger_table).to_string() + ) - def __new__(cls, index: int, number_of_devices: int, medium: Medium): # device #0 is the client # device #1 is a disconnected node @@ -192,8 +228,9 @@ def __new__(cls, index: int, number_of_devices: int, medium: Medium): if index == 1: return ChordNode(index, number_of_devices, medium, False, None) if index > 1: - return ChordNode(index, number_of_devices, medium, True, all_routing_data[index-2]) - + return ChordNode( + index, number_of_devices, medium, True, all_routing_data[index - 2] + ) class QuitMessage(MessageStub): @@ -201,7 +238,8 @@ def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'QUIT REQUEST {self.source} -> {self.destination}' + return f"QUIT REQUEST {self.source} -> {self.destination}" + class PutMessage(MessageStub): def __init__(self, sender: int, destination: int, guid: int, data: str): @@ -210,7 +248,8 @@ def __init__(self, sender: int, destination: int, guid: int, data: str): self.data = data def __str__(self): - return f'PUT MESSAGE {self.source} -> {self.destination}: ({self.guid}, {self.data})' + return f"PUT MESSAGE {self.source} -> {self.destination}: ({self.guid}, {self.data})" + class GetReqMessage(MessageStub): def __init__(self, sender: int, destination: int, guid: int): @@ -218,45 +257,54 @@ def __init__(self, sender: int, destination: int, guid: int): self.guid = guid def __str__(self): - return f'GET REQUEST MESSAGE {self.source} -> {self.destination}: ({self.guid})' + return f"GET REQUEST MESSAGE {self.source} -> {self.destination}: ({self.guid})" -class GetReqMessage(MessageStub): + +class GetRspMessage(MessageStub): def __init__(self, sender: int, destination: int, guid: int, data: str): super().__init__(sender, destination) self.guid = guid self.data = data def __str__(self): - return f'GET RESPONSE MESSAGE {self.source} -> {self.destination}: ({self.guid}, {self.data})' + return f"GET RESPONSE MESSAGE {self.source} -> {self.destination}: ({self.guid}, {self.data})" + class StartJoinMessage(MessageStub): def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'StartJoinMessage MESSAGE {self.source} -> {self.destination}' + return f"StartJoinMessage MESSAGE {self.source} -> {self.destination}" + class JoinReqMessage(MessageStub): - def __init__(self, sender: int, destination: int): # etc + def __init__(self, sender: int, destination: int): # etc super().__init__(sender, destination) + def __str__(self): - return f'JoinReqMessage MESSAGE {self.source} -> {self.destination}' + return f"JoinReqMessage MESSAGE {self.source} -> {self.destination}" + class JoinRspMessage(MessageStub): - def __init__(self, sender: int, destination: int): # etc + def __init__(self, sender: int, destination: int): # etc super().__init__(sender, destination) + def __str__(self): - return f'JoinRspMessage MESSAGE {self.source} -> {self.destination}' + return f"JoinRspMessage MESSAGE {self.source} -> {self.destination}" + class NotifyMessage(MessageStub): - def __init__(self, sender: int, destination: int): # etc + def __init__(self, sender: int, destination: int): # etc super().__init__(sender, destination) + def __str__(self): - return f'NotifyMessage MESSAGE {self.source} -> {self.destination}' + return f"NotifyMessage MESSAGE {self.source} -> {self.destination}" + class StabilizeMessage(MessageStub): - def __init__(self, sender: int, destination: int): # etc + def __init__(self, sender: int, destination: int): # etc super().__init__(sender, destination) - def __str__(self): - return f'StabilizeMessage MESSAGE {self.source} -> {self.destination}' + def __str__(self): + return f"StabilizeMessage MESSAGE {self.source} -> {self.destination}" \ No newline at end of file diff --git a/exercises/exercise12.py b/exercises/exercise12.py index d024cdc..8b68947 100644 --- a/exercises/exercise12.py +++ b/exercises/exercise12.py @@ -1,15 +1,11 @@ -import math import random -import sys import threading +from typing import Optional from emulators.Device import Device from emulators.Medium import Medium from emulators.MessageStub import MessageStub -import json -import time - # if you need controlled repetitions: # random.seed(100) @@ -29,51 +25,54 @@ class AodvNode(Device): def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) # I get the topology from the singleton - self.neighbors = TopologyCreator.get_topology(number_of_devices, probability_arc)[index] + self.neighbors = TopologyCreator.get_topology( + number_of_devices, probability_arc + )[index] # I initialize the "routing tables". Feel free to create your own structure if you prefer - self.reverse_path = {} - self.forward_path = {} - self.bcast_ids = [] + self.forward_path: dict[ + int, int + ] = {} # "Destination index" --> "Next-hop index" + self.reverse_path: dict[int, int] = {} # "Source index" --> "Next-hop index" + self.bcast_ids = [] # Type hint left out on purpose due to tasks below # data structures to cache outgoing messages, and save received data - self.saved_data = [] - self.outgoing_message_cache = [] + self.saved_data: list[str] = [] + self.outgoing_message_cache: list[DataMessage] = [] def run(self): - last = random.randint(0, self.number_of_devices() - 1) + last = random.randint(0, self.number_of_devices - 1) # I send the message to myself, so it gets routed - message = DataMessage(self.index(), self.index(), last, f"hi I am {self.index()}") - self.medium().send(message) + message = DataMessage(self.index, self.index, last, f"Hi. I am {self.index}.") + self.medium.send(message) while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() - def next_hop(self, last): - for destination in self.forward_path: - if destination == last: - return self.forward_path[destination] - return None + def next_hop(self, last: int) -> Optional[int]: + return self.forward_path.get(last) # Returns "None" if key does not exist def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, DataMessage): - if self.index() == ingoing.last: + if self.index == ingoing.last: # the message is for me self.saved_data.append(ingoing.data) # AodvNode.messages_lock.acquire() AodvNode.data_messages_received += 1 AodvNode.messages_lock.release() - if AodvNode.data_messages_received == self.number_of_devices(): - for i in range(0, self.number_of_devices()): - self.medium().send(QuitMessage(self.index(), i)) + if AodvNode.data_messages_received == self.number_of_devices: + for i in range(0, self.number_of_devices): + self.medium.send(QuitMessage(self.index, i)) # else: - next = self.next_hop(ingoing.last) # change self.next_hop if you implement a different data structure for the routing tables + next = self.next_hop( + ingoing.last + ) # change self.next_hop if you implement a different data structure for the routing tables if next is not None: # I know how to reach the destination - message = DataMessage(self.index(), next, ingoing.last, ingoing.data) - self.medium().send(message) + message = DataMessage(self.index, next, ingoing.last, ingoing.data) + self.medium.send(message) return True # I don't have the route to the destination. # I need to save the outgoing message in a cache @@ -90,7 +89,7 @@ def handle_ingoing(self, ingoing: MessageStub): # TODO pass - if self.index() == ingoing.last: + if self.index == ingoing.last: # the message is for me. I can send back a Route Reply # TODO pass @@ -102,7 +101,7 @@ def handle_ingoing(self, ingoing: MessageStub): # If I don't have the forward_path in my routing table, I save it # TODO - if self.index() == ingoing.first: + if self.index == ingoing.first: # Finally, I can send all the messages I had saved "first -> last" from my cache # TODO pass @@ -115,22 +114,20 @@ def handle_ingoing(self, ingoing: MessageStub): return True def print_result(self): - print(f"AODV node {self.index()} quits, neighbours {self.neighbors}, forward paths {self.forward_path}, reverse paths {self.reverse_path}, saved data {self.saved_data} length of message cache (should be 0) {len(self.outgoing_message_cache)}") - - + print( + f"AODV node {self.index} quits: neighbours = {self.neighbors}, forward paths = {self.forward_path}, reverse paths = {self.reverse_path}, saved data = {self.saved_data}, length of message cache (should be 0) = {len(self.outgoing_message_cache)}" + ) class TopologyCreator: # singleton design pattern - __topology = None + __topology: dict[int, list[int]] = None # "Node index" --> [neighbor node indices] - def __check_connected(topology): + def __check_connected(topology: dict[int, list[int]]) -> Optional[tuple[int, int]]: # if the network is connected, it returns None; # if not, it returns two nodes belonging to two different partitions - queue = [] - visited = [] - visited.append(0) - queue.append(0) + queue = [0] + visited = [0] while queue: s = queue.pop(0) for neigh in topology.get(s): @@ -142,8 +139,8 @@ def __check_connected(topology): return (visited[-1], n) return None - def __create_topology(number_of_devices, probability): - topology = {} + def __create_topology(number_of_devices: int, probability: float): + topology: dict[int, list[int]] = {} for i in range(0, number_of_devices): topology[i] = [] for i in range(0, number_of_devices): @@ -158,22 +155,21 @@ def __create_topology(number_of_devices, probability): return topology @classmethod - def get_topology(cls, number_of_devices, probability): - if cls.__topology is not None: - return cls.__topology - - cls.__topology = TopologyCreator.__create_topology(number_of_devices, probability) + def get_topology(cls, number_of_devices: int, probability: float): + if cls.__topology is None: + cls.__topology = TopologyCreator.__create_topology( + number_of_devices, probability + ) return cls.__topology - - class QuitMessage(MessageStub): def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'QUIT REQUEST {self.source} -> {self.destination}' + return f"QUIT REQUEST {self.source} -> {self.destination}" + class AodvRreqMessage(MessageStub): def __init__(self, sender: int, destination: int, first: int, last: int): @@ -182,7 +178,8 @@ def __init__(self, sender: int, destination: int, first: int, last: int): self.last = last def __str__(self): - return f'RREQ MESSAGE {self.source} -> {self.destination}: ({self.first} -> {self.last})' + return f"RREQ MESSAGE {self.source} -> {self.destination}: ({self.first} -> {self.last})" + class AodvRrepMessage(MessageStub): def __init__(self, sender: int, destination: int, first: int, last: int): @@ -191,7 +188,7 @@ def __init__(self, sender: int, destination: int, first: int, last: int): self.last = last def __str__(self): - return f'RREP MESSAGE {self.source} -> {self.destination}: ({self.first} -> {self.last})' + return f"RREP MESSAGE {self.source} -> {self.destination}: ({self.first} -> {self.last})" class DataMessage(MessageStub): @@ -201,4 +198,4 @@ def __init__(self, sender: int, destination: int, last: int, data: str): self.data = data def __str__(self): - return f'DATA MESSAGE {self.source} -> {self.destination}: (final target {self.last} data {self.data})' + return f'DATA MESSAGE {self.source} -> {self.destination}: (final target = {self.last}, data = "{self.data}")' diff --git a/exercises/exercise2.py b/exercises/exercise2.py index 6595a0b..c025382 100644 --- a/exercises/exercise2.py +++ b/exercises/exercise2.py @@ -9,70 +9,85 @@ def __init__(self, sender: int, destination: int, table): self.table = table def __str__(self): - return f'RipMessage: {self.source} -> {self.destination} : {self.table}' + return f"RipMessage: {self.source} -> {self.destination} : {self.table}" + class RoutableMessage(MessageStub): - def __init__(self, sender: int, destination: int, first_node: int, last_node: int, content): + def __init__( + self, sender: int, destination: int, first_node: int, last_node: int, content + ): super().__init__(sender, destination) self.content = content self.first_node = first_node self.last_node = last_node def __str__(self): - return f'RoutableMessage: {self.source} -> {self.destination} : {self.content}' - - + return f"RoutableMessage: {self.source} -> {self.destination} : {self.content}" class RipCommunication(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) - - self.neighbors = [] # generate an appropriate list + + self.neighbors = [] # generate an appropriate list self.routing_table = dict() def run(self): for neigh in self.neighbors: self.routing_table[neigh] = (neigh, 1) - self.routing_table[self.index()] = (self.index(), 0) + self.routing_table[self.index] = (self.index, 0) for neigh in self.neighbors: - self.medium().send(RipMessage(self.index(), neigh, self.routing_table)) + self.medium.send(RipMessage(self.index, neigh, self.routing_table)) while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is None: # this call is only used for synchronous networks - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() continue if type(ingoing) is RipMessage: - print(f"Device {self.index()}: Got new table from {ingoing.source}") + print(f"Device {self.index}: Got new table from {ingoing.source}") returned_table = self.merge_tables(ingoing.source, ingoing.table) if returned_table is not None: self.routing_table = returned_table for neigh in self.neighbors: - self.medium().send(RipMessage(self.index(), neigh, self.routing_table)) + self.medium.send( + RipMessage(self.index, neigh, self.routing_table) + ) if type(ingoing) is RoutableMessage: - print(f"Device {self.index()}: Routing from {ingoing.first_node} to {ingoing.last_node} via #{self.index()}: [#{ingoing.content}]") - if ingoing.last_node is self.index(): - print(f"\tDevice {self.index()}: delivered message from {ingoing.first_node} to {ingoing.last_node}: {ingoing.content}") + print( + f"Device {self.index}: Routing from {ingoing.first_node} to {ingoing.last_node} via #{self.index}: [#{ingoing.content}]" + ) + if ingoing.last_node is self.index: + print( + f"\tDevice {self.index}: delivered message from {ingoing.first_node} to {ingoing.last_node}: {ingoing.content}" + ) continue if self.routing_table[ingoing.last_node] is not None: (next_hop, distance) = self.routing_table[ingoing.last_node] - self.medium().send(RoutableMessage(self.index(), next_hop, ingoing.first_node, ingoing.last_node, ingoing.content)) + self.medium.send( + RoutableMessage( + self.index, + next_hop, + ingoing.first_node, + ingoing.last_node, + ingoing.content, + ) + ) continue - print(f"\tDevice {self.index()}: DROP Unknown route #{ingoing.first_node} to #{ingoing.last_node} via #{self.index}, message #{ingoing.content}") + print( + f"\tDevice {self.index}: DROP Unknown route #{ingoing.first_node} to #{ingoing.last_node} via #{self.index}, message #{ingoing.content}" + ) # this call is only used for synchronous networks - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def merge_tables(self, src, table): # return None if the table does not change pass - def print_result(self): - print(f'\tDevice {self.index()} has routing table: {self.routing_table}') \ No newline at end of file + print(f"\tDevice {self.index} has routing table: {self.routing_table}") diff --git a/exercises/exercise4.py b/exercises/exercise4.py index 9c07302..04ceeed 100644 --- a/exercises/exercise4.py +++ b/exercises/exercise4.py @@ -1,7 +1,4 @@ import math -import random -import threading -import time from emulators.Medium import Medium from emulators.Device import Device, WorkerDevice @@ -14,18 +11,17 @@ def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'Ping: {self.source} -> {self.destination}' + return f"Ping: {self.source} -> {self.destination}" class Pinger(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) self._output_ping = False def run(self): while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if isinstance(ingoing, Ping): if self._output_ping: print("Ping") @@ -33,7 +29,7 @@ def run(self): else: print("Pong") self._output_ping = True - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def print_result(self): print("Done!") @@ -46,7 +42,6 @@ class Type(enum.Enum): class MutexMessage(MessageStub): - def __init__(self, sender: int, destination: int, message_type: Type): super().__init__(sender, destination) self._type = message_type @@ -62,11 +57,11 @@ def is_release(self): def __str__(self): if self._type == Type.REQUEST: - return f'Request: {self.source} -> {self.destination}' + return f"Request: {self.source} -> {self.destination}" if self._type == Type.RELEASE: - return f'Release: {self.source} -> {self.destination}' + return f"Release: {self.source} -> {self.destination}" if self._type == Type.GRANT: - return f'Grant: {self.source} -> {self.destination}' + return f"Grant: {self.source} -> {self.destination}" class Centralised: @@ -78,61 +73,59 @@ def __new__(cls, index: int, number_of_devices: int, medium: Medium): class Coordinator(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) - assert (self.index() == 0) # we assume that the coordinator is fixed at index 0. + assert self.index == 0 # we assume that the coordinator is fixed at index 0. self._granted = None self._waiting = [] def run(self): while True: while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is None: break if ingoing.is_request(): self._waiting.append(ingoing.source) elif ingoing.is_release(): - assert (self._granted == ingoing.source) + assert self._granted == ingoing.source self._granted = None if len(self._waiting) > 0 and self._granted is None: self._granted = self._waiting.pop(0) - self.medium().send(MutexMessage(self.index(), self._granted, Type.GRANT)) - self.medium().wait_for_next_round() + self.medium.send( + MutexMessage(self.index, self._granted, Type.GRANT) + ) + self.medium.wait_for_next_round() def print_result(self): print("Coordinator Terminated") class Requester(WorkerDevice): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) - assert (self.index() != 0) # we assume that the coordinator is fixed at index 0. + assert self.index != 0 # we assume that the coordinator is fixed at index 0. self._requested = False def run(self): while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is not None: if ingoing.is_grant(): - assert (self._requested) + assert self._requested self.do_work() self._requested = False - self.medium().send( - MutexMessage(self.index(), 0, Type.RELEASE)) + self.medium.send(MutexMessage(self.index, 0, Type.RELEASE)) - if self.has_work() and not self._requested: + if self.has_work and not self._requested: self._requested = True - self.medium().send( - MutexMessage(self.index(), 0, Type.REQUEST)) + self.medium.send(MutexMessage(self.index, 0, Type.REQUEST)) - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def print_result(self): - print(f"Requester {self.index()} Terminated with request? {self._requested}") + print(f"Requester {self.index} Terminated with request? {self._requested}") class TokenRing(WorkerDevice): @@ -145,25 +138,24 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): def run(self): while True: while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is None: break if ingoing.is_grant(): self._has_token = True if self._has_token: - if self.has_work(): + if self.has_work: self.do_work() - nxt = (self.index() + 1) % self.number_of_devices() + nxt = (self.index + 1) % self.number_of_devices self._has_token = False - self.medium().send(MutexMessage(self.index(), nxt, Type.GRANT)) - self.medium().wait_for_next_round() + self.medium.send(MutexMessage(self.index, nxt, Type.GRANT)) + self.medium.wait_for_next_round() def print_result(self): - print(f"Token Ring {self.index()} Terminated with request? {self._requested}") + print(f"Token Ring {self.index} Terminated with request? {self._requested}") class StampedMessage(MutexMessage): - def __init__(self, sender: int, destination: int, message_type: Type, time: int): super().__init__(sender, destination, message_type) self._stamp = time @@ -172,7 +164,7 @@ def stamp(self) -> int: return self._stamp def __str__(self): - return super().__str__() + f' [stamp={self._stamp}]' + return super().__str__() + f" [stamp={self._stamp}]" class State(enum.Enum): @@ -182,7 +174,6 @@ class State(enum.Enum): class RicartAgrawala(WorkerDevice): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) self._state = State.RELEASED @@ -192,10 +183,10 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): def run(self): while True: - if self.has_work(): + if self.has_work: self.acquire() while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is not None: if ingoing.is_grant(): self.handle_grant(ingoing) @@ -203,23 +194,27 @@ def run(self): self.handle_request(ingoing) else: break - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_request(self, message: StampedMessage): new_time = max(self._time, message.stamp()) + 1 - if self._state == State.HELD or (self._state == State.WANTED and - (self._time, self.index()) < (message.stamp(), message.source)): + if self._state == State.HELD or ( + self._state == State.WANTED + and (self._time, self.index) < (message.stamp(), message.source) + ): self._time = new_time self._waiting.append(message.source) else: new_time += 1 - self.medium().send(StampedMessage(self.index(), message.source, Type.GRANT, new_time)) + self.medium.send( + StampedMessage(self.index, message.source, Type.GRANT, new_time) + ) self._time = new_time def handle_grant(self, message: StampedMessage): self._grants += 1 self._time = max(self._time, message.stamp()) + 1 - if self._grants == self.number_of_devices() - 1: + if self._grants == self.number_of_devices - 1: self._state = State.HELD self.do_work() self.release() @@ -229,9 +224,7 @@ def release(self): self._state = State.RELEASED self._time += 1 for id in self._waiting: - self.medium().send( - StampedMessage(self.index(), id, Type.GRANT, self._time) - ) + self.medium.send(StampedMessage(self.index, id, Type.GRANT, self._time)) self._waiting.clear() def acquire(self): @@ -239,18 +232,17 @@ def acquire(self): return self._state = State.WANTED self._time += 1 - for id in self.medium().ids(): - if id != self.index(): - self.medium().send( - StampedMessage(self.index(), id, - Type.REQUEST, self._time)) + for id in self.medium.ids: + if id != self.index: + self.medium.send( + StampedMessage(self.index, id, Type.REQUEST, self._time) + ) def print_result(self): - print(f"RA {self.index()} Terminated with request? {self._state == State.WANTED}") + print(f"RA {self.index} Terminated with request? {self._state == State.WANTED}") class Maekawa(WorkerDevice): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) self._state = State.RELEASED @@ -259,20 +251,20 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): self._grants = 0 # Generate quorums/ voting set self._voting_set = set() - dimension = int(math.sqrt(self.number_of_devices())) - offset_x = self.index() % dimension - offset_y = int(self.index() / dimension) + dimension = int(math.sqrt(self.number_of_devices)) + offset_x = self.index % dimension + offset_y = int(self.index / dimension) for i in range(0, dimension): # same "column" - if i * dimension + offset_x < self.number_of_devices(): + if i * dimension + offset_x < self.number_of_devices: self._voting_set.add(i * dimension + offset_x) # same "row" - if offset_y * dimension + i < self.number_of_devices(): + if offset_y * dimension + i < self.number_of_devices: self._voting_set.add(offset_y * dimension + i) def run(self): while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is not None: if ingoing.is_grant(): self.handle_grant(ingoing) @@ -280,16 +272,16 @@ def run(self): self.handle_request(ingoing) elif ingoing.is_release(): self.handle_release(ingoing) - if self.has_work(): + if self.has_work: self.acquire() - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def acquire(self): if self._state == State.WANTED: return self._state = State.WANTED for id in self._voting_set: - self.medium().send(MutexMessage(self.index(), id, Type.REQUEST)) + self.medium.send(MutexMessage(self.index, id, Type.REQUEST)) def handle_grant(self, message: MutexMessage): self._grants += 1 @@ -302,25 +294,25 @@ def release(self): self._grants = 0 self._state = State.RELEASED for id in self._voting_set: - self.medium().send(MutexMessage(self.index(), id, Type.RELEASE)) + self.medium.send(MutexMessage(self.index, id, Type.RELEASE)) def handle_request(self, message: MutexMessage): if self._state == State.HELD or self._voted: self._waiting.append(message.source) else: self._voted = True - self.medium().send(MutexMessage(self.index(), message.source, Type.GRANT)) + self.medium.send(MutexMessage(self.index, message.source, Type.GRANT)) def handle_release(self, message: MutexMessage): if len(self._waiting) > 0: nxt = self._waiting.pop(0) self._voted = True - self.medium().send(MutexMessage(self.index(), nxt, Type.GRANT)) + self.medium.send(MutexMessage(self.index, nxt, Type.GRANT)) else: self._voted = False def print_result(self): - print(f"MA {self.index()} Terminated with request? {self._state == State.WANTED}") + print(f"MA {self.index} Terminated with request? {self._state == State.WANTED}") class SKToken(MessageStub): @@ -336,13 +328,14 @@ def ln(self): return self._ln def __str__(self): - return f"Token: {self.source} -> {self.destination}, \n" \ - f"\t\tQueue {str(self._queue)}\n" \ - f"\t\tLN {str(self._ln)}" + return ( + f"Token: {self.source} -> {self.destination}, \n" + f"\t\tQueue {str(self._queue)}\n" + f"\t\tLN {str(self._ln)}" + ) class SuzukiKasami(WorkerDevice): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) self._rn = [0 for _ in range(0, number_of_devices)] @@ -358,7 +351,7 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): def run(self): while True: self.handle_messages() - if self.has_work(): + if self.has_work: if self._token is not None: self._working = True self.do_work() @@ -368,11 +361,11 @@ def run(self): else: self.acquire() - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_messages(self): while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is None: break if isinstance(ingoing, SKToken): @@ -386,15 +379,15 @@ def handle_request(self, message: StampedMessage): (queue, ln) = self._token if self._rn[message.source] == ln[message.source] + 1: self._token = None - self.medium().send(SKToken(self.index(), message.source, queue, ln)) + self.medium.send(SKToken(self.index, message.source, queue, ln)) def release(self): self._working = False self._requested = False (queue, ln) = self._token - ln[self.index()] = self._rn[self.index()] + ln[self.index] = self._rn[self.index] # let's generate a new queue with all devices with outstanding requests - for id in self.medium().ids(): + for id in self.medium.ids: if ln[id] + 1 == self._rn[id]: if id not in queue: queue.append(id) @@ -402,24 +395,25 @@ def release(self): if len(queue) > 0: nxt = queue.pop(0) self._token = None - self.medium().send(SKToken(self.index(), nxt, queue, ln)) + self.medium.send(SKToken(self.index, nxt, queue, ln)) def acquire(self): if self._requested: return # Tell everyone that we want the token! self._requested = True - self._rn[self.index()] += 1 - for id in self.medium().ids(): - if id != self.index(): - self.medium().send( - StampedMessage(self.index(), id, Type.REQUEST, self._rn[self.index()])) + self._rn[self.index] += 1 + for id in self.medium.ids: + if id != self.index: + self.medium.send( + StampedMessage(self.index, id, Type.REQUEST, self._rn[self.index]) + ) # Election Algorithms -class Vote(MessageStub): +class Vote(MessageStub): def __init__(self, sender: int, destination: int, vote: int, decided: bool): super().__init__(sender, destination) self._vote = vote @@ -432,7 +426,7 @@ def decided(self): return self._decided def __str__(self): - return f'Vote: {self.source} -> {self.destination}, voted for {self._vote}, decided? {self._decided}' + return f"Vote: {self.source} -> {self.destination}, voted for {self._vote}, decided? {self._decided}" class ChangRoberts(Device): @@ -443,37 +437,35 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): def run(self): while True: - nxt = (self.index() + 1) % self.number_of_devices() + nxt = (self.index + 1) % self.number_of_devices if not self._participated: - self.medium().send( - Vote(self.index(), nxt, self.index(), False)) + self.medium.send(Vote(self.index, nxt, self.index, False)) self._participated = True - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is not None: - if ingoing.vote() == self.index(): + if ingoing.vote() == self.index: if not ingoing.decided(): - self.medium().send( - Vote(self.index(), nxt, self.index(), True)) + self.medium.send(Vote(self.index, nxt, self.index, True)) else: - self._leader = self.index() + self._leader = self.index return # this device is the new leader - elif ingoing.vote() < self.index(): + elif ingoing.vote() < self.index: continue - elif ingoing.vote() > self.index(): + elif ingoing.vote() > self.index: # forward the message - self.medium().send( - Vote(self.index(), nxt, ingoing.vote(), ingoing.decided())) + self.medium.send( + Vote(self.index, nxt, ingoing.vote(), ingoing.decided()) + ) if ingoing.decided(): self._leader = ingoing.vote() return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def print_result(self): - print(f'Leader seen from {self._id} is {self._leader}') + print(f"Leader seen from {self._id} is {self._leader}") class Bully(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) self._leader = None @@ -481,7 +473,7 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): self._election = False def largest(self): - return self.index() == max(self.medium().ids()) + return self.index == max(self.medium.ids) def run(self): first_round = True @@ -492,11 +484,18 @@ def run(self): new_election = False while True: - ingoing = self.medium().receive() + ingoing = self.medium.receive() if ingoing is not None: got_input = True - if ingoing.vote() < self.index(): - self.medium().send(Vote(self.index(), ingoing.source, self.index(), self.largest())) + if ingoing.vote() < self.index: + self.medium.send( + Vote( + self.index, + ingoing.source, + self.index, + self.largest(), + ) + ) new_election = True else: self._shut_up = True @@ -515,20 +514,20 @@ def run(self): self.start_election() else: # we are the new leader, we could declare everybody else dead - for id in self.medium().ids(): - if id != self.index(): - self.medium().send(Vote(self.index(), id, self.index(), True)) - self._leader = self.index() + for id in self.medium.ids: + if id != self.index: + self.medium.send(Vote(self.index, id, self.index, True)) + self._leader = self.index return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() first_round = False def start_election(self): if not self._election: self._election = True - for id in self.medium().ids(): - if id > self.index(): - self.medium().send(Vote(self.index(), id, self.index(), self.largest())) + for id in self.medium.ids: + if id > self.index: + self.medium.send(Vote(self.index, id, self.index, self.largest())) def print_result(self): - print(f'Leader seen from {self._id} is {self._leader}') + print(f"Leader seen from {self._id} is {self._leader}") diff --git a/exercises/exercise5.py b/exercises/exercise5.py index fef20f7..c846a86 100644 --- a/exercises/exercise5.py +++ b/exercises/exercise5.py @@ -2,13 +2,13 @@ import threading import time -import math import copy from emulators.Device import Device from emulators.Medium import Medium from emulators.MessageStub import MessageStub + class MulticastMessage(MessageStub): def __init__(self, sender: int, destination: int, content): super().__init__(sender, destination) @@ -18,7 +18,7 @@ def content(self): return self._content def __str__(self): - return f'Multicast: {self.source} -> {self.destination} [{self._content}]' + return f"Multicast: {self.source} -> {self.destination} [{self._content}]" class MulticastListener: @@ -56,8 +56,13 @@ def forward(self, message): class BasicMulticast(Device, MulticastService): - - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: MulticastListener = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: MulticastListener = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application @@ -67,12 +72,12 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium, applicati def run(self): while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): self.handle_ingoing(ingoing) while len(self._outbox) > 0: msg = self._outbox.pop(0) self.send_to_all(msg) - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, MulticastMessage): @@ -81,10 +86,10 @@ def handle_ingoing(self, ingoing: MessageStub): self._application.forward(ingoing) def send_to_all(self, content): - for id in self.medium().ids(): + for id in self.medium.ids: # we purposely send to ourselves also! - message = MulticastMessage(self.index(), id, content) - self.medium().send(message) + message = MulticastMessage(self.index, id, content) + self.medium.send(message) def send(self, message): self._outbox.append(copy.deepcopy(message)) @@ -94,26 +99,33 @@ def print_result(self): class ReliableMulticast(MulticastListener, MulticastService, Device): - - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: MulticastListener = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: MulticastListener = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application else: self._application = Multicaster(index, self) self._b_multicast = BasicMulticast(index, number_of_devices, medium, self) - self._seq_number = 0 # not strictly needed, but helps giving messages a unique ID + self._seq_number = ( + 0 # not strictly needed, but helps giving messages a unique ID + ) self._received = set() def send(self, content): - self._b_multicast.send((self.index(), self._seq_number, content)) + self._b_multicast.send((self.index, self._seq_number, content)) self._seq_number += 1 def deliver(self, message): (origin_index, seq_number, content) = message if message not in self._received: - if origin_index is not self.index(): + if origin_index is not self.index: self._b_multicast.send(message) self._received.add(message) self._application.deliver(content) @@ -134,7 +146,7 @@ def seq_number(self): return self._seq_number def __str__(self): - return f'NACK: {self.source} -> {self.destination}: {self._seq_number}' + return f"NACK: {self.source} -> {self.destination}: {self._seq_number}" class Resend(MessageStub): @@ -146,19 +158,24 @@ def message(self): return self._message def __str__(self): - return f'Resend: {self.source} -> {self.destination}: {self._message}' + return f"Resend: {self.source} -> {self.destination}: {self._message}" class ReliableIPMulticast(MulticastListener, MulticastService, Device): - - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: MulticastListener = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: MulticastListener = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application else: self._application = Multicaster(index, self) self._b_multicast = BasicMulticast(index, number_of_devices, medium, self) - self._seq_numbers = [0 for _ in medium.ids()] + self._seq_numbers = [0 for _ in medium.ids] self._received = {} def deliver(self, message): @@ -170,9 +187,8 @@ def deliver(self, message): self.nack_missing(seq_numbers) def send(self, content): - self._received[(self.index(), - self._seq_numbers[self.index()])] = content - self._b_multicast.send((self.index(), self._seq_numbers, content)) + self._received[(self.index, self._seq_numbers[self.index])] = content + self._b_multicast.send((self.index, self._seq_numbers, content)) self.try_deliver() def run(self): @@ -180,9 +196,17 @@ def run(self): def forward(self, message): if isinstance(message, NACK): - self.medium().send(Resend(self.index(), message.source, - (self.index(), self._seq_numbers, - self._received[(self.index(), message.seq_number())]))) + self.medium.send( + Resend( + self.index, + message.source, + ( + self.index, + self._seq_numbers, + self._received[(self.index, message.seq_number())], + ), + ) + ) elif isinstance(message, Resend): self.deliver(message.message()) else: @@ -199,8 +223,7 @@ def try_deliver(self): def nack_missing(self, n_seq: list[int]): for id in range(0, len(n_seq)): for mid in range(self._seq_numbers[id] + 1, n_seq[id]): - self.medium().send( - NACK(self.index(), id, mid)) + self.medium.send(NACK(self.index, id, mid)) class Order: @@ -215,11 +238,17 @@ def message_id(self): return self._message_id def __str__(self): - return f'Order(<{self.message_id()}> = {self.order()})' + return f"Order(<{self.message_id()}> = {self.order()})" -class TOSEQMulticast(MulticastListener, MulticastService, Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: MulticastListener = None): +class TOSEQMulticast(MulticastListener, MulticastService, Device): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: MulticastListener = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application @@ -232,14 +261,14 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium, applicati self._received = {} def send(self, content): - self._b_multicast.send((self.index(), self._l_seq, content)) + self._b_multicast.send((self.index, self._l_seq, content)) self._l_seq += 1 def deliver(self, message): if not isinstance(message, Order): (sid, sseq, content) = message mid = (sid, sseq) - if self.index() == 0: + if self.index == 0: # index 0 is global sequencer self._order[mid] = self._g_seq self._b_multicast.send(Order(mid, self._g_seq)) @@ -248,7 +277,7 @@ def deliver(self, message): else: self._received[mid] = content self.try_deliver() - elif self.index() != 0: + elif self.index != 0: # index 0 is global sequencer self._order[message.message_id()] = message.order() self.try_deliver() @@ -269,7 +298,9 @@ def forward(self, message): class Vote(MessageStub): - def __init__(self, sender: int, destination: int, order: (int, int), message_id: (int, int)): + def __init__( + self, sender: int, destination: int, order: (int, int), message_id: (int, int) + ): super().__init__(sender, destination) self._order = order self._message_id = message_id @@ -281,12 +312,17 @@ def message_id(self) -> (int, int): return self._message_id def __str__(self): - return f'Vote: {self.source} -> {self.destination}: <{self.message_id()}> = {self.order()}' + return f"Vote: {self.source} -> {self.destination}: <{self.message_id()}> = {self.order()}" class ISISMulticast(MulticastListener, MulticastService, Device): - - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: MulticastListener = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: MulticastListener = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application @@ -305,8 +341,8 @@ def run(self): self._b_multicast.run() def send(self, content): - self._b_multicast.send((self.index(), self._l_seq, content)) - self._votes[(self.index(), self._l_seq)] = [] + self._b_multicast.send((self.index, self._l_seq, content)) + self._votes[(self.index, self._l_seq)] = [] self._l_seq += 1 def deliver(self, message): @@ -319,18 +355,14 @@ def deliver(self, message): self._hb_q[(sid, sseq)] = content self._p_seq = max(self._a_seq, self._p_seq) + 1 # We should technically send proposer ID for tie-breaks - self.medium().send( - Vote(self.index(), sid, self._p_seq, (sid, sseq)) - ) + self.medium.send(Vote(self.index, sid, self._p_seq, (sid, sseq))) def forward(self, message): if isinstance(message, Vote): votes = self._votes[message.message_id()] votes.append(message.order()) - if len(votes) == self.number_of_devices(): - self._b_multicast.send( - Order(message.message_id(), max(votes)) - ) + if len(votes) == self.number_of_devices: + self._b_multicast.send(Order(message.message_id(), max(votes))) else: self._application.forward(message) @@ -344,20 +376,25 @@ def try_deliver(self): class COMulticast(MulticastListener, MulticastService, Device): - - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: MulticastListener = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: MulticastListener = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application else: self._application = Multicaster(index, self) self._b_multicast = BasicMulticast(index, number_of_devices, medium, self) - self._n_vect = [-1 for _ in self.medium().ids()] + self._n_vect = [-1 for _ in self.medium.ids] self._hb_q = [] def send(self, content): - self._n_vect[self.index()] += 1 - self._b_multicast.send((self._n_vect, self.index(), content)) + self._n_vect[self.index] += 1 + self._b_multicast.send((self._n_vect, self.index, content)) def deliver(self, message): self._hb_q.append(message) @@ -367,7 +404,7 @@ def forward(self, message): self._application.forward(message) def try_deliver(self): - for (vec, index, content) in self._hb_q: + for vec, index, content in self._hb_q: if self.is_next(vec, index): self._application.deliver(content) self._n_vect[index] += 1 @@ -376,7 +413,7 @@ def try_deliver(self): def is_next(self, vec, index): if vec[index] != self._n_vect[index] + 1: return False - for i in self.medium().ids(): + for i in self.medium.ids: if i != index and vec[i] > self._n_vect[i]: return False return True diff --git a/exercises/exercise6.py b/exercises/exercise6.py index 48233da..bdc373e 100644 --- a/exercises/exercise6.py +++ b/exercises/exercise6.py @@ -15,7 +15,6 @@ def initial_value(self): class SimpleRequester(ConsensusRequester): - def __init__(self): self._proposal = random.randint(0, 100) @@ -32,7 +31,8 @@ def consensus_reached(self, element): SimpleRequester._consensus = element if SimpleRequester._consensus != element: raise ValueError( - f"Disagreement in consensus, expected {element} but other process already got {SimpleRequester._consensus}") + f"Disagreement in consensus, expected {element} but other process already got {SimpleRequester._consensus}" + ) class Propose(MessageStub): @@ -44,11 +44,17 @@ def value(self): return self._value def __str__(self): - return f'Propose: {self.source} -> {self.destination}: {self._value}' + return f"Propose: {self.source} -> {self.destination}: {self._value}" class FResilientConsensus(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: ConsensusRequester = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: ConsensusRequester = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application @@ -60,10 +66,10 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium, applicati def run(self): self.b_multicast(Propose({self._application.initial_value})) - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() for i in range(0, self._f): # f + 1 rounds v_p = self._v.copy() - for p in self.medium().receive_all(): + for p in self.medium.receive_all(): self._v.update(p.value()) if i != self._f - 1: self.b_multicast(Propose(v_p.difference(v_p))) @@ -71,18 +77,23 @@ def run(self): self._application.consensus_reached(min(self._v)) def b_multicast(self, message: MessageStub): - message.source = self.index() - for i in self.medium().ids(): + message.source = self.index + for i in self.medium.ids: message.destination = i - self.medium().send(message) + self.medium.send(message) def print_result(self): - print(f"Device {self.index()} agrees on {min(self._v)}") + print(f"Device {self.index} agrees on {min(self._v)}") class SingleByzantine(Device): - - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: ConsensusRequester = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: ConsensusRequester = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application @@ -91,7 +102,7 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium, applicati self._consensus = None def run(self): - if self.index() == 0: + if self.index == 0: self.run_commander() else: self.run_lieutenant() @@ -101,27 +112,27 @@ def run_commander(self): """Done!""" def run_lieutenant(self): - self.medium().wait_for_next_round() - from_commander = self.medium().receive_all() + self.medium.wait_for_next_round() + from_commander = self.medium.receive_all() assert len(from_commander) <= 1 v = None if from_commander is not None and len(from_commander) > 0: v = from_commander[0].value() - self.b_multicast(Propose((self.index(), v))) - self.medium().wait_for_next_round() - from_others = [m.value() for m in self.medium().receive_all()] + self.b_multicast(Propose((self.index, v))) + self.medium.wait_for_next_round() + from_others = [m.value() for m in self.medium.receive_all()] self._consensus = find_majority(from_others) self._application.consensus_reached(self._consensus) def b_multicast(self, message: MessageStub): - message.source = self.index() - for i in self.medium().ids(): + message.source = self.index + for i in self.medium.ids: message.destination = i - self.medium().send(message) + self.medium.send(message) def print_result(self): - if self.index() != 0: - print(f"Device {self.index()} is done, consensus: {self._consensus}") + if self.index != 0: + print(f"Device {self.index} is done, consensus: {self._consensus}") else: print("Commander is done") @@ -140,16 +151,54 @@ def find_majority(raw: [(int, int)]): best = None return best + +def most_common(): + raise NotImplementedError() + + class King(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: ConsensusRequester = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: ConsensusRequester = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application else: self._application = SimpleRequester() + def b_multicast(self, message: MessageStub): + message.source = self.index + for i in self.medium.ids: + message.destination = i + self.medium.send(message) + def run(self): - pass + # Set own v to a preferred value // f+1 phases in total + # for each phase i ∈ 0 . . . f do + # // round 1: + # B-multicast(v ) + # Await vj from each process pj + # Set v to the most frequent element ∈ v0...vn−1 + # Set mult to the number of occurrences of v + # Set v to a default value if mult < n/2 + # // round 2: + # if k = i then + # B-multicast(v ) // king for phase k is pk , send tie breaker + # end + # Set vk to the value received from the king + # if mult ≤ (n/2) + f then + # Set v to the vk + # end + # end + v = random.randint(1, 100) + for i in range(0, self.number_of_devices): + self.b_multicast(message=Propose(v)) + # vs = self.medium.receive_all() + # ... def print_result(self): pass @@ -161,19 +210,22 @@ def __init__(self, sender: int, destination: int, uid: int): self.uid = uid def __str__(self): - return f'PREPARE {self.source} -> {self.destination}: {self.uid}' + return f"PREPARE {self.source} -> {self.destination}: {self.uid}" class PromiseMessage(MessageStub): - def __init__(self, sender: int, destination: int, uid: int, prev_uid: int, prev_value): + def __init__( + self, sender: int, destination: int, uid: int, prev_uid: int, prev_value + ): super().__init__(sender, destination) self.uid = uid self.prev_uid = prev_uid self.prev_value = prev_value def __str__(self): - return f'PROMISE {self.source} -> {self.destination}: {self.uid}' + \ - ('' if self.prev_uid == 0 else f'accepted {self.prev_uid}, {self.prev_value}') + return f"PROMISE {self.source} -> {self.destination}: {self.uid}" + ( + "" if self.prev_uid == 0 else f"accepted {self.prev_uid}, {self.prev_value}" + ) class RequestAcceptMessage(MessageStub): @@ -183,7 +235,7 @@ def __init__(self, sender: int, destination: int, uid: int, value): self.value = value def __str__(self): - return f'ACCEPT-REQUEST {self.source} -> {self.destination}: {self.uid}, {self.value}' + return f"ACCEPT-REQUEST {self.source} -> {self.destination}: {self.uid}, {self.value}" class AcceptMessage(MessageStub): @@ -193,11 +245,13 @@ def __init__(self, sender: int, destination: int, uid: int, value): self.value = value def __str__(self): - return f'ACCEPT {self.source} -> {self.destination}: {self.uid}, {self.value}' + return f"ACCEPT {self.source} -> {self.destination}: {self.uid}, {self.value}" class PAXOSNetwork: - def __init__(self, index: int, medium: Medium, acceptors: list[int], learners: list[int]): + def __init__( + self, index: int, medium: Medium, acceptors: list[int], learners: list[int] + ): self._acceptors = acceptors self._learners = learners self._medium = medium @@ -236,14 +290,20 @@ def index(self): class PAXOS(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium, application: ConsensusRequester = None): + def __init__( + self, + index: int, + number_of_devices: int, + medium: Medium, + application: ConsensusRequester = None, + ): super().__init__(index, number_of_devices, medium) if application is not None: self._application = application else: self._application = SimpleRequester() # assumes everyone has every role - config = PAXOSNetwork(index, self.medium(), self.medium().ids(), self.medium().ids()) + config = PAXOSNetwork(index, self.medium, self.medium.ids, self.medium.ids) self._proposer = Proposer(config, self._application) self._acceptor = Acceptor(config) self._learner = Learner(config, self._application) @@ -253,9 +313,9 @@ def run(self): self._proposer.check_prepare() if self._proposer.done() and self._acceptor.done() and self._learner.done(): return - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): self.handle_ingoing(ingoing) - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, PrepareMessage): @@ -337,7 +397,7 @@ def handle_accept(self, msg: AcceptMessage): if self._done: return self._done = True - print(f'CONSENSUS {self._network.index} LEARNER on {msg.value}') + print(f"CONSENSUS {self._network.index} LEARNER on {msg.value}") self._application.consensus_reached(msg.value) def done(self): diff --git a/exercises/exercise7.py b/exercises/exercise7.py index cabbf72..e722cd6 100644 --- a/exercises/exercise7.py +++ b/exercises/exercise7.py @@ -1,15 +1,9 @@ -import math -import random -import threading -import time - from emulators.Medium import Medium from emulators.Device import Device from emulators.MessageStub import MessageStub class Vote(MessageStub): - def __init__(self, sender: int, destination: int, vote: int, decided: bool): super().__init__(sender, destination) self._vote = vote @@ -22,11 +16,10 @@ def decided(self): return self._decided def __str__(self): - return f'Vote: {self.source} -> {self.destination}, voted for {self._vote}, decided? {self._decided}' + return f"Vote: {self.source} -> {self.destination}, voted for {self._vote}, decided? {self._decided}" class Bully(Device): - def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) self._leader = None @@ -34,7 +27,7 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): self._election = False def largest(self): - return self.index() == max(self.medium().ids()) + return self.index == max(self.medium.ids) def run(self): """TODO""" @@ -43,4 +36,4 @@ def start_election(self): """TODO""" def print_result(self): - print(f'Leader seen from {self._id} is {self._leader}') \ No newline at end of file + print(f"Leader seen from {self._id} is {self._leader}") diff --git a/exercises/exercise8.py b/exercises/exercise8.py index 374ba75..4e85ad2 100644 --- a/exercises/exercise8.py +++ b/exercises/exercise8.py @@ -1,6 +1,4 @@ -import math import random -import sys from emulators.Device import Device from emulators.Medium import Medium @@ -12,92 +10,84 @@ # if you need repetition: # random.seed(100) + class GfsMaster(Device): def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) - self._metadata = {} - self.chunks_being_allocated = [] + self._metadata: dict[ + tuple[str, int], tuple[int, list[int]] + ] = {} # (filename, chunk_index) -> (chunkhandle, [chunkservers]) + self.chunks_being_allocated: list[ + tuple[int, int] + ] = [] # [(chunkhandle, requester_index)] GfsNetwork.gfsmaster.append(index) def run(self): - # since this is a server, its job is to answer for requests (messages), then do something + # since this is a server, its job is to wait for requests (messages), then do something while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, File2ChunkReqMessage): key = (ingoing.filename, ingoing.chunkindex) chunk = self._metadata.get(key) if chunk is not None: - for c in self.chunks_being_allocated: if c[0] == chunk[0]: self.chunks_being_allocated.append((chunk[0], ingoing.source)) return True - - - anwser = File2ChunkRspMessage( - self.index(), - ingoing.source, - chunk[0], - chunk[1] - ) - self.medium().send(anwser) + answer = File2ChunkRspMessage( + self.index, ingoing.source, chunk[0], chunk[1] + ) + self.medium.send(answer) else: if ingoing.createIfNotExists: - self.do_allocate_request(ingoing.filename, ingoing.chunkindex, ingoing.source) + self.do_allocate_request( + ingoing.filename, ingoing.chunkindex, ingoing.source + ) else: - anwser = File2ChunkRspMessage( - self.index(), - ingoing.source, - 0, - [] - ) - self.medium().send(anwser) + answer = File2ChunkRspMessage(self.index, ingoing.source, 0, []) + self.medium.send(answer) elif isinstance(ingoing, QuitMessage): - print("I am Master " + str(self.index()) + " and I am quitting") + print(f"I am Master {self.index} and I am quitting") return False elif isinstance(ingoing, AllocateChunkRspMessage): - if not ingoing.result == "ok": - print("allocation failed! I am quitting!!") + if ingoing.result != "ok": + print("Allocation failed! I am quitting!!") return False for chunk in self._metadata.values(): if chunk[0] == ingoing.chunkhandle: self.add_chunk_to_metadata(chunk, ingoing.source) return True - def add_chunk_to_metadata(self, chunk, chunkserver): + def add_chunk_to_metadata(self, chunk: tuple[int, list[int]], chunkserver: int): chunk[1].append(chunkserver) if len(chunk[1]) == NUMBER_OF_REPLICAS: - for request in self.chunks_being_allocated: - if request[0] == chunk[0]: - anwser = File2ChunkRspMessage( - self.index(), - request[1], - chunk[0], - chunk[1] - ) - self.medium().send(anwser) - - - def do_allocate_request(self, filename, chunkindex, requester): + requests = [ + request + for request in self.chunks_being_allocated + if request[0] == chunk[0] + ] + for request in requests: + answer = File2ChunkRspMessage( + self.index, request[1], chunk[0], chunk[1] + ) + self.medium.send(answer) + self.chunks_being_allocated.remove(request) + + def do_allocate_request(self, filename, chunkindex: int, requester: int): chunkhandle = random.randint(0, 999999) self.chunks_being_allocated.append((chunkhandle, requester)) self._metadata[(filename, chunkindex)] = (chunkhandle, []) - chunkservers = [] - startnumber = random.randint(0, 9999) - for i in range(NUMBER_OF_REPLICAS): - numChunkServer = len(GfsNetwork.gfschunkserver) - chosen = GfsNetwork.gfschunkserver[(startnumber + i ) % numChunkServer] - chunkservers.append(chosen) - + # Allocate the new chunk on "NUMBER_OF_REPLICAS" random chunkservers + chunkservers = random.sample(GfsNetwork.gfschunkserver, NUMBER_OF_REPLICAS) for i in chunkservers: - message = AllocateChunkReqMessage(self.index(), i, chunkhandle, chunkservers) - self.medium().send(message) + message = AllocateChunkReqMessage(self.index, i, chunkhandle, chunkservers) + self.medium.send(message) def print_result(self): pass @@ -107,81 +97,84 @@ class GfsChunkserver(Device): def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) GfsNetwork.gfschunkserver.append(index) - self.localchunks = {} + self.localchunks: dict[int, str] = {} # chunkhandle -> contents # the first server in chunkservers is the primary - self.chunkservers = {} + self.chunkservers: dict[int, list[int]] = {} # chunkhandle -> [chunkservers] def run(self): # since this is a server, its job is to answer for requests (messages), then do something while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, QuitMessage): - print("I am Chunk Server " + str(self.index()) + " and I am quitting") + print(f"I am Chunk Server {self.index} and I am quitting") return False elif isinstance(ingoing, AllocateChunkReqMessage): self.do_allocate_chunk(ingoing.chunkhandle, ingoing.chunkservers) - message = AllocateChunkRspMessage(self.index(), ingoing.source, ingoing.chunkhandle, "ok") - self.medium().send(message) + message = AllocateChunkRspMessage( + self.index, ingoing.source, ingoing.chunkhandle, "ok" + ) + self.medium.send(message) elif isinstance(ingoing, RecordAppendReqMessage): # - # TODO: need to implement the storate operation - # do not forget the passive replication discipline + # TODO: need to implement the storage operation + # do not forget the passive replication discipline # pass return True - def do_allocate_chunk(self, chunkhandle, servers): + def do_allocate_chunk(self, chunkhandle: int, servers: list[int]): self.localchunks[chunkhandle] = "" self.chunkservers[chunkhandle] = servers def print_result(self): print("chunk server quit. Currently, my saved chunks are as follows:") - for c in self.localchunks: - print("chunk " + str(c) + " : " + str(self.localchunks[c])) + for chunkhandle, contents in self.localchunks.items(): + print(f"chunk {chunkhandle} : {contents}") class GfsClient(Device): def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) - if index == 0: - for i in range(5): - pass def run(self): - # being a client, it listens to incoming messags, but it also does something to put the ball rolling - print("i am client " + str(self.index())) + # being a client, it listens to incoming messages, but it also does something to put the ball rolling + print(f"I am Client {self.index}") master = GfsNetwork.gfsmaster[0] - message = File2ChunkReqMessage(self.index(), master, "myfile.txt", 0, True) - self.medium().send(message) + message = File2ChunkReqMessage(self.index, master, "myfile.txt", 0, True) + self.medium.send(message) while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, File2ChunkRspMessage): - print("I found out where my chunk is: " + str(ingoing.chunkhandle) + " locations: " + str(ingoing.locations)) + print( + f"I found out where my chunk is: {ingoing.chunkhandle}, locations: {ingoing.locations}" + ) # I select a random chunk server, and I send the append request # I do not necessarily select the primary - - randomserver = ingoing.locations[random.randint(0,999)%len(ingoing.locations)] - data = "hello from client number " + str(self.index()) + "\n" - self.medium().send(RecordAppendReqMessage(self.index(), randomserver, ingoing.chunkhandle, data)) + randomserver = random.choice(ingoing.locations) + data = f"hello from client number {self.index}\n" + self.medium.send( + RecordAppendReqMessage( + self.index, randomserver, ingoing.chunkhandle, data + ) + ) elif isinstance(ingoing, RecordAppendRspMessage): # project completed, time to quit for i in GfsNetwork.gfsmaster: - self.medium().send(QuitMessage(self.index(), i)) + self.medium.send(QuitMessage(self.index, i)) for i in GfsNetwork.gfschunkserver: - self.medium().send(QuitMessage(self.index(), i)) - + self.medium.send(QuitMessage(self.index, i)) return False return True @@ -189,7 +182,6 @@ def print_result(self): pass - class GfsNetwork: def __new__(cls, index: int, number_of_devices: int, medium: Medium): if index < NUMBER_OF_MASTERS: @@ -198,49 +190,60 @@ def __new__(cls, index: int, number_of_devices: int, medium: Medium): return GfsChunkserver(index, number_of_devices, medium) else: return GfsClient(index, number_of_devices, medium) + gfsmaster = [] gfschunkserver = [] - - class QuitMessage(MessageStub): def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'QUIT REQUEST {self.source} -> {self.destination}' - + return f"QUIT REQUEST {self.source} -> {self.destination}" class File2ChunkReqMessage(MessageStub): - def __init__(self, sender: int, destination: int, filename: str, chunkindex: int, createIfNotExists = False): + def __init__( + self, + sender: int, + destination: int, + filename: str, + chunkindex: int, + createIfNotExists=False, + ): super().__init__(sender, destination) self.filename = filename self.chunkindex = chunkindex self.createIfNotExists = createIfNotExists def __str__(self): - return f'FILE2CHUNK REQUEST {self.source} -> {self.destination}: ({self.filename}, {self.chunkindex}, createIfNotExists = {self.createIfNotExists})' + return f"FILE2CHUNK REQUEST {self.source} -> {self.destination}: ({self.filename}, {self.chunkindex}, createIfNotExists = {self.createIfNotExists})" + class File2ChunkRspMessage(MessageStub): - def __init__(self, sender: int, destination: int, chunkhandle: int, locations: list): + def __init__( + self, sender: int, destination: int, chunkhandle: int, locations: list + ): super().__init__(sender, destination) self.chunkhandle = chunkhandle self.locations = locations def __str__(self): - return f'FILE2CHUNK RESPONSE {self.source} -> {self.destination}: ({self.chunkhandle}, {self.locations})' + return f"FILE2CHUNK RESPONSE {self.source} -> {self.destination}: ({self.chunkhandle}, {self.locations})" class AllocateChunkReqMessage(MessageStub): - def __init__(self, sender: int, destination: int, chunkhandle: int, chunkservers: list): + def __init__( + self, sender: int, destination: int, chunkhandle: int, chunkservers: list[int] + ): super().__init__(sender, destination) self.chunkhandle = chunkhandle self.chunkservers = chunkservers def __str__(self): - return f'ALLOCATE REQUEST {self.source} -> {self.destination}: ({self.chunkhandle})' + return f"ALLOCATE REQUEST {self.source} -> {self.destination}: ({self.chunkhandle})" + class AllocateChunkRspMessage(MessageStub): def __init__(self, sender: int, destination: int, chunkhandle: int, result: str): @@ -249,7 +252,8 @@ def __init__(self, sender: int, destination: int, chunkhandle: int, result: str) self.result = result def __str__(self): - return f'ALLOCATE RESPONSE {self.source} -> {self.destination}: ({self.chunkhandle, self.result})' + return f"ALLOCATE RESPONSE {self.source} -> {self.destination}: ({self.chunkhandle, self.result})" + class RecordAppendReqMessage(MessageStub): def __init__(self, sender: int, destination: int, chunkhandle: int, data: str): @@ -258,7 +262,8 @@ def __init__(self, sender: int, destination: int, chunkhandle: int, data: str): self.data = data def __str__(self): - return f'RECORD APPEND REQUEST {self.source} -> {self.destination}: ({self.chunkhandle}, {self.data})' + return f"RECORD APPEND REQUEST {self.source} -> {self.destination}: ({self.chunkhandle}, {self.data})" + class RecordAppendRspMessage(MessageStub): def __init__(self, sender: int, destination: int, result: str): @@ -267,5 +272,11 @@ def __init__(self, sender: int, destination: int, result: str): # TODO: possibly, complete this message with the fields you need def __str__(self): - return f'RECORD APPEND REQUEST {self.source} -> {self.destination}: ({self.result})' + return f"RECORD APPEND RESPONSE {self.source} -> {self.destination}: ({self.result})" +class HeartbeatMessage(MessageStub): + def __init__(self, sender: int, destination: int): + super().__init__(sender, destination) + + def __str__(self): + return f"HEARTBEAT {self.source} -> {self.destination}" \ No newline at end of file diff --git a/exercises/exercise9.py b/exercises/exercise9.py index 62e9260..e156b13 100644 --- a/exercises/exercise9.py +++ b/exercises/exercise9.py @@ -1,6 +1,3 @@ -import math -import random -import sys import os from enum import Enum @@ -9,59 +6,77 @@ from emulators.MessageStub import MessageStub - class Role(Enum): # is the Worker a Mapper, a Reducer, or in Idle state? IDLE = 1 MAPPER = 2 REDUCER = 3 + class MapReduceMaster(Device): def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) + self.number_partitions: int = -1 + self.result_files: list[str] = [] + self.number_finished_reducers = 0 def run(self): # since this is a server, its job is to answer for requests (messages), then do something while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, ClientJobStartMessage): # I assign ingoing.number_partitions workers as reducers, the rest as mappers # and I assign some files to each mapper self.number_partitions = ingoing.number_partitions - number_of_mappers = self.number_of_devices() - self.number_partitions - 2 + number_of_mappers = self.number_of_devices - self.number_partitions - 2 for i in range(2, self.number_partitions + 2): - message = ReduceTaskMessage(self.index(), i, i - 2, self.number_partitions, number_of_mappers) # the reducer needs to know how many mappers they are, to know when its task is completed - self.medium().send(message) + message = ReduceTaskMessage( + self.index, i, i - 2, self.number_partitions, number_of_mappers + ) # the reducer needs to know how many mappers they are, to know when its task is completed + self.medium.send(message) for i in range(0, number_of_mappers): length = len(ingoing.filenames) - length = 5 # TODO: comment out this line to process all files, once you think your code is ready + length = 5 # TODO: comment out this line to process all files, once you think your code is ready first = int(length * i / number_of_mappers) - last = int(length * (i+1) / number_of_mappers) - message = MapTaskMessage(self.index(), self.number_partitions + 2 + i, ingoing.filenames[first:last], self.number_partitions) - self.medium().send(message) + last = int(length * (i + 1) / number_of_mappers) + message = MapTaskMessage( + self.index, + self.number_partitions + 2 + i, + ingoing.filenames[first:last], + self.number_partitions, + ) + self.medium.send(message) elif isinstance(ingoing, QuitMessage): # if the client is satisfied with the work done, I can tell all workers to quit, then I can quit for w in MapReduceNetwork.workers: - self.medium().send(QuitMessage(self.index(), w)) + self.medium.send(QuitMessage(self.index, w)) return False elif isinstance(ingoing, MappingDoneMessage): - # TODO: - # contact all reducers, telling them that a mapper has completed its job - # hint: you need to define a new message type, for example ReducerVisitMapperMessage + # TODO: contact all reducers, telling them that a mapper has completed its job + # hint: you need to define a new message type, for example ReducerVisitMapperMessage (see MapReduceWorker) pass elif isinstance(ingoing, ReducingDoneMessage): - # I can tell the client that the job is done - message = ClientJobCompletedMessage(1, 0) - self.medium().send(message) + self.number_finished_reducers += 1 + self.result_files.append(ingoing.result_filename) + if self.number_finished_reducers == self.number_partitions: + # I can tell the client that the job is done + message = ClientJobCompletedMessage( + 1, MapReduceNetwork.client_index, self.result_files + ) + self.medium.send(message) return True def print_result(self): - print("Master " + str(self.index()) + " quits") + print(f"Master {self.index} quits") + + +class ReducerVisitMapperMessage: + pass class MapReduceWorker(Device): @@ -72,44 +87,46 @@ def __init__(self, index: int, number_of_devices: int, medium: Medium): # number of partitions (equals to number of reducers) self.number_partitions = 0 # variables if it is a mapper - self.M_files_to_process = {} # list of files to process - self.M_cached_results = {} # in-memory cache - self.M_stored_results = {} # "R" files containing results + self.M_files_to_process: list[str] = [] # list of files to process + self.M_cached_results: dict[str, int] = {} # in-memory cache + self.M_stored_results: dict[ + int, dict[str, int] + ] = {} # "R" files containing results. partition -> word -> count # variables if it is a reducer - self.R_my_partition = 0 # the partition I am managing - self.R_number_mappers = 0 # how many mappers there are. I need to know it to decide when I can tell the master I am done with the reduce task - + self.R_my_partition = 0 # the partition I am managing + self.R_number_mappers = 0 # how many mappers there are. I need to know it to decide when I can tell the master I am done with the reduce task - def mapper_process_file(self, filename): + def mapper_process_file(self, filename: str) -> dict[str, int]: # goal: return the occurrences of words in the file words = [] - with open("ex9data/books/"+filename) as file: + with open("ex9data/books/" + filename) as file: for line in file: - words+=line.split() + words += line.split() result = {} for word in words: result[word.lower()] = 1 + result.get(word.lower(), 0) return result - def mapper_partition_function(self, key): - # compute the partition based on the key (see the lecture material) - # this function should be supplied by the client We stick to a fixed function for sake of clarity + def mapper_partition_function(self, key: str) -> int: + # Compute the partition based on the key (see the lecture material) + # This function should be supplied by the client, but we stick to a fixed function for sake of clarity char = ord(key[0]) - if char < ord('a'): - char = ord('a') - if char > ord('z'): - char = ord('z') - partition = (char - ord('a')) * self.number_partitions / (1+ord('z')-ord('a')) + if char < ord("a"): + char = ord("a") + if char > ord("z"): + char = ord("z") + partition = ( + (char - ord("a")) * self.number_partitions / (1 + ord("z") - ord("a")) + ) return int(partition) - def mapper_shuffle(self): # goal: merge all the data I have in the cache to the stored results WITH SHUFFLE, then flush the cache for word in self.M_cached_results: p = self.mapper_partition_function(word) old_value = self.M_stored_results[p].get(word, 0) self.M_stored_results[p][word] = self.M_cached_results[word] + old_value - self.M_cached_results = [] # flushing the cache + self.M_cached_results = [] # flushing the cache def do_some_work(self): if self.role == Role.IDLE: @@ -121,32 +138,34 @@ def do_some_work(self): # if I have no more files, I "store" it locally into partitions and tell the master that I am done if self.M_files_to_process != []: filename = self.M_files_to_process.pop() - print(f"mapper {self.index()} file {filename} processed") map_result = self.mapper_process_file(filename) - for k in map_result: - self.M_cached_results[k] = self.M_cached_results.get(k, 0) + map_result[k] + for word in map_result: + self.M_cached_results[word] = ( + self.M_cached_results.get(word, 0) + map_result[word] + ) + print(f"Mapper {self.index}: file '{filename}' processed") if self.M_files_to_process == []: self.mapper_shuffle() - message = MappingDoneMessage(self.index(), 0) - self.medium().send(message) + message = MappingDoneMessage( + self.index, MapReduceNetwork.master_index + ) + self.medium.send(message) if self.role == Role.REDUCER: # not much to do: everything is done when the master tells us about a mapper that completed its task pass - - def run(self): # since this is a worker, it looks for incoming requests (messages), then it works a little while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return self.do_some_work() - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, QuitMessage): - print("I am Mapper " + str(self.index()) + " and I am quitting") + print(f"I am Worker {self.index} and I am quitting") return False elif isinstance(ingoing, MapTaskMessage): # I was assigned to be a mapper, thus I: @@ -165,80 +184,85 @@ def handle_ingoing(self, ingoing: MessageStub): self.R_my_partition = ingoing.my_partition self.R_number_mappers = ingoing.number_mappers # nothing to do until the Master tells us to contact Mappers - pass - elif isinstance(ingoing, ReducerVisitMapperMessage): + elif isinstance( + ingoing, ReducerVisitMapperMessage + ): # 'ReducerVisitMapperMessage' does not exist by default # the master is saying that a mapper is done # thus this reducer will: - # get the "stored" results for the mapper, for the correct partition + # get the "stored" results from the mapper, for the correct partition (new message type) # if it is the last mapper I have to contact, I will: # merge the data - # store it somewhere + # store resulting data in "ex9data/results/{my_partition_file_name}" # tell the master I am done # TODO: write the code pass return True def print_result(self): - print(f"worker quits. It was a {self.Role}") + print(f"Worker {self.index} quits. It was a {self.role}") class MapReduceClient(Device): def __init__(self, index: int, number_of_devices: int, medium: Medium): super().__init__(index, number_of_devices, medium) + self.result_files: list[str] = [] def scan_for_books(self): - books = [] - with os.scandir('ex9data/books/') as entries: - for entry in entries: - if entry.is_file() and entry.name.endswith(".txt"): - books.append(entry.name) - return books + with os.scandir("ex9data/books/") as entries: + return [ + entry.name + for entry in entries + if entry.is_file() and entry.name.endswith(".txt") + ] def run(self): # being a client, it listens to incoming messages, but it also does something to put the ball rolling - print("i am client " + str(self.index())) + print(f"I am client {self.index}") books = self.scan_for_books() - - message = ClientJobStartMessage(self.index(), 1, books, 3) # TODO: experiment with different number of reducers - self.medium().send(message) + + message = ClientJobStartMessage( + self.index, MapReduceNetwork.master_index, books, 3 + ) # TODO: experiment with different number of reducers + self.medium.send(message) while True: - for ingoing in self.medium().receive_all(): + for ingoing in self.medium.receive_all(): if not self.handle_ingoing(ingoing): return - self.medium().wait_for_next_round() + self.medium.wait_for_next_round() def handle_ingoing(self, ingoing: MessageStub): if isinstance(ingoing, ClientJobCompletedMessage): # I can tell the master to quit # I will print the result later, with the print_result function - self.medium().send(QuitMessage(self.index(), 1)) + self.medium.send(QuitMessage(self.index, MapReduceNetwork.master_index)) + self.result_files = ingoing.result_files return False return True def print_result(self): for filename in self.result_files: - print("results from file: {self.filename}") - with open("ex9data/results/" + filename) as file: + print(f"Results from file: {filename}") + with open(f"ex9data/results/{filename}") as file: for line in file: print("\t" + line.rstrip()) - class MapReduceNetwork: def __new__(cls, index: int, number_of_devices: int, medium: Medium): # client has index 0 # master has index 1 # workers have index 2+ - cls.workers = [] - if index == 0: + if index == MapReduceNetwork.client_index: return MapReduceClient(index, number_of_devices, medium) - elif index == 1: + elif index == MapReduceNetwork.master_index: return MapReduceMaster(index, number_of_devices, medium) else: return MapReduceWorker(index, number_of_devices, medium) - + client_index = 0 + master_index = 1 + workers: list[int] = [] class QuitMessage(MessageStub): @@ -246,18 +270,20 @@ def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'QUIT REQUEST {self.source} -> {self.destination}' - + return f"QUIT REQUEST {self.source} -> {self.destination}" class ClientJobStartMessage(MessageStub): - def __init__(self, sender: int, destination: int, filenames: list, number_partitions: int): + def __init__( + self, sender: int, destination: int, filenames: list, number_partitions: int + ): super().__init__(sender, destination) self.filenames = filenames self.number_partitions = number_partitions def __str__(self): - return f'CLIENT START JOB REQUEST {self.source} -> {self.destination}: ({len(self.filenames)} files, {self.number_partitions} partitions)' + return f"CLIENT START JOB REQUEST {self.source} -> {self.destination}: ({len(self.filenames)} files, {self.number_partitions} partitions)" + class ClientJobCompletedMessage(MessageStub): def __init__(self, sender: int, destination: int, result_files: list): @@ -265,44 +291,51 @@ def __init__(self, sender: int, destination: int, result_files: list): self.result_files = result_files def __str__(self): - return f'CLIENT JOB COMPLETED {self.source} -> {self.destination} ({self.result_files})' - + return f"CLIENT JOB COMPLETED {self.source} -> {self.destination} ({self.result_files})" class MapTaskMessage(MessageStub): - def __init__(self, sender: int, destination: int, filenames: list, number_partitions: int): + def __init__( + self, sender: int, destination: int, filenames: list, number_partitions: int + ): super().__init__(sender, destination) self.filenames = filenames self.number_partitions = number_partitions def __str__(self): - return f'MAP TASK ASSIGNMENT {self.source} -> {self.destination}: ({len(self.filenames)} files, {self.number_partitions} partitions)' - + return f"MAP TASK ASSIGNMENT {self.source} -> {self.destination}: ({len(self.filenames)} files, {self.number_partitions} partitions)" + + class MappingDoneMessage(MessageStub): def __init__(self, sender: int, destination: int): super().__init__(sender, destination) def __str__(self): - return f'MAP TASK COMKPLETED {self.source} -> {self.destination}' - + return f"MAP TASK COMPLETED {self.source} -> {self.destination}" class ReduceTaskMessage(MessageStub): - def __init__(self, sender: int, destination: int, my_partition: int, number_partitions: int, number_mappers: int): + def __init__( + self, + sender: int, + destination: int, + my_partition: int, + number_partitions: int, + number_mappers: int, + ): super().__init__(sender, destination) self.my_partition = my_partition self.number_partitions = number_partitions self.number_mappers = number_mappers def __str__(self): - return f'REDUCE TASK ASSIGNMENT {self.source} -> {self.destination}: (partition is {self.my_partition}, {self.number_partitions} partitions, {self.number_mappers} mappers)' + return f"REDUCE TASK ASSIGNMENT {self.source} -> {self.destination}: (partition is {self.my_partition}, {self.number_partitions} partitions, {self.number_mappers} mappers)" + class ReducingDoneMessage(MessageStub): - def __init__(self, sender: int, destination: int): + def __init__(self, sender: int, destination: int, result_filename: str): super().__init__(sender, destination) + self.result_filename = result_filename def __str__(self): - return f'REDUCE TASK COMPLETED {self.source} -> {self.destination}: ()' - - - + return f"REDUCE TASK COMPLETED {self.source} -> {self.destination}: result_filename = {self.result_filename}" diff --git a/figures/stepping_gui.png b/figures/stepping_gui.png new file mode 100644 index 0000000..1df4a09 Binary files /dev/null and b/figures/stepping_gui.png differ diff --git a/icon.ico b/icon.ico new file mode 100644 index 0000000..b138c53 Binary files /dev/null and b/icon.ico differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..689b6f7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +cryptography >= 37.0.4 +PyQt6 >= 6.3.1 +pynput >= 1.7.6 \ No newline at end of file