diff --git a/app/db_update.py b/app/db_update.py index 83aa22e..81e0f06 100644 --- a/app/db_update.py +++ b/app/db_update.py @@ -12,7 +12,7 @@ print('mode must be prod or dev') exit() -if args.version != 'add_github' and args.version != 'add_constructicon' and args.version != 'all' and args.version != 'new_roles': +if args.version != 'add_github' and args.version != 'add_constructicon' and args.version != 'all' and args.version != 'new_roles' and args.version != 'add_validated_tree': print('version must be add_github, add_constructicon or all') exit() @@ -49,6 +49,18 @@ def migrate_add_constructicon(engine): def migrate_new_roles(engine): with engine.connect() as connection: connection.execute('UPDATE projectaccess SET access_level = 3 WHERE projectaccess.access_level == 2') + + +def migrate_add_validated_tree(engine): + with engine.connect() as connection: + connection.execute('ALTER TABLE projects RENAME COLUMN exercise_mode To blind_annotation_mode') + connection.execute('ALTER TABLE projects ADD config STRING') + connection.execute('ALTER TABLE projects ADD language STRING') + connection.execute('ALTER TABLE projects DROP show_all_trees') + connection.execute('ALTER TABLE exerciselevel RENAME COLUMN exercise_level TO blind_annotation_level') + connection.execute('ALTER TABLE exerciselevel RENAME TO blindannotationlevel') + connection.execute('CREATE TABLE user_tags (id INTEGER NOT NULL, user_id VARCHAR(256), tags JSONB, PRIMARY KEY(id), FOREIGN KEY(user_id) REFERENCES users(id))') + if args.version == 'add_github': migrate_add_github(engine) @@ -61,4 +73,7 @@ def migrate_new_roles(engine): migrate_add_constructicon(engine) if args.version == 'new_roles': - migrate_new_roles(engine) \ No newline at end of file + migrate_new_roles(engine) + +if args.version == 'add_validated_tree': + migrate_add_validated_tree(engine) \ No newline at end of file diff --git a/app/github/service.py b/app/github/service.py index cccf80b..7584e1f 100644 --- a/app/github/service.py +++ b/app/github/service.py @@ -8,11 +8,13 @@ from typing import List from flask import abort +from flask_login import current_user from app import db from app.config import Config from app.projects.service import ProjectService from app.utils.grew_utils import GrewService, grew_request , SampleExportService +from app.user.service import UserService import app.samples.service as SampleService from .model import GithubRepository, GithubCommitStatus @@ -30,7 +32,7 @@ def get_github_synchronized_repository(project_id): @staticmethod def synchronize_github_repository(user_id, project_id, repository_name, branch, sha): - github_repository ={ + github_repository = { "project_id": project_id, "user_id": user_id, "repository_name": repository_name, @@ -220,8 +222,7 @@ def delete_file_from_github(access_token, project_name, full_name, sample_name): def delete_sample_from_project(project_name, sample_name): project = ProjectService.get_by_name(project_name) GrewService.delete_sample(project_name, sample_name) - SampleService.SampleRoleService.delete_by_sample_name(project.id, sample_name) - SampleService.SampleExerciseLevelService.delete_by_sample_name(project.id, sample_name) + SampleService.SampleBlindAnnotationLevelService.delete_by_sample_name(project.id, sample_name) class GithubService: @@ -334,7 +335,11 @@ def create_tree(access_token, full_name, updated_samples, project_name, username tree = [] sample_names, sample_content_files = GrewService.get_samples_with_string_contents(project_name, updated_samples) for sample_name, sample in zip(sample_names,sample_content_files): - content = sample.get(username) + if (username == 'validated'): + owner_username = UserService.get_by_id(current_user.id).username + content = GrewService.get_validated_trees_filled_up_with_owner_trees(project_name, sample_name, owner_username) + else: + content = sample.get(username) sha = GithubService.create_blob_for_updated_file(access_token, full_name, content) blob = {"path": sample_name+".conllu", "mode":"100644", "type":"blob", "sha": sha} tree.append(blob) diff --git a/app/grew/controller.py b/app/grew/controller.py index ab58977..acd5722 100644 --- a/app/grew/controller.py +++ b/app/grew/controller.py @@ -59,25 +59,38 @@ def post(self, project_name: str): class SearchResource(Resource): "Search" def post(self, project_name: str): + parser = reqparse.RequestParser() parser.add_argument(name="pattern", type=str) parser.add_argument(name="userType", type=str) args = parser.parse_args() - pattern = args.get("pattern") - user_type = args.get("userType") + trees_type = args.get("userType") + user_type = 'all' if trees_type == 'pending' else trees_type reply = GrewService.search_pattern_in_graphs(project_name, pattern, user_type) + if reply["status"] != "OK": abort(400) + trees = {} for m in reply["data"]: if m["user_id"] == "": abort(409) conll = m["conll"] trees = formatTrees_new(m, trees, conll) - return trees + + search_results = {} + if trees_type == "pending": + for sample_name, sample_results in trees.items(): + search_results[sample_name] = { + sent_id: result for sent_id, result in sample_results.items() if 'validated' not in result["conlls"].keys() + } + else: + search_results = trees + + return search_results @api.route("//sample//search") @@ -98,8 +111,15 @@ def post(self, project_name: str, sample_name: str): args = parser.parse_args() pattern = args.get("pattern") - user_type = args.get("userType") + trees_type = args.get("userType") + + user_type = 'all' if trees_type == 'pending' else trees_type + reply = GrewService.search_pattern_in_graphs(project_name, pattern, user_type) + + if reply["status"] != "OK": + abort(400) + trees = {} for m in reply["data"]: if m["sample_id"] != sample_name: @@ -108,7 +128,16 @@ def post(self, project_name: str, sample_name: str): abort(409) conll = m["conll"] trees = formatTrees_new(m, trees, conll) - return trees + + search_results = {} + if trees_type == 'pending': + search_results[sample_name] = { + sent_id: result for sent_id, result in trees[sample_name].items() if 'validated' not in result["conlls"].keys() + } + else: + search_results = trees + + return search_results @api.route("//try-package") class TryPackageResource(Resource): @@ -158,6 +187,8 @@ def post(self, project_name): user_ids = { "one": [current_user.username, "__last__"] } elif tableType=='recent': user_ids = { "one": ["__last__"] } + elif tableType=='validated': + user_ids = { "one": ["validated"]} elif tableType=='all': user_ids = "all" reply = grew_request( @@ -267,7 +298,6 @@ def post_process_diffs(grew_search_results, other_users, features): matches[other_user_id] = list_matches conlls[other_user_id] = grew_search_results[sample_name][sent_id]["conlls"][other_user_id] if len(conlls) > 0 : - print('test') post_processed_results[sample_name][sent_id] = { "sentence": grew_search_results[sample_name][sent_id]["sentence"], "sample_name": sample_name, diff --git a/app/lexicon/controller.py b/app/lexicon/controller.py index 3e58290..c1b231d 100644 --- a/app/lexicon/controller.py +++ b/app/lexicon/controller.py @@ -30,12 +30,14 @@ def post(self, project_name: str): lexicon_type=args.get("lexiconType") prune=args.get("prune") - if lexicon_type=='user': + if lexicon_type == 'user': user_ids = { "one": [current_user.username] } - elif lexicon_type=='user_recent': + elif lexicon_type == 'user_recent': user_ids = { "one": [current_user.username, "__last__"] } - elif lexicon_type=='recent': + elif lexicon_type == 'recent': user_ids = { "one": ["__last__"] } + elif lexicon_type == 'validated': + user_ids = { "one": ["validated"] } elif lexicon_type=='all': user_ids = "all" diff --git a/app/projects/controller.py b/app/projects/controller.py index 5213f6f..632f1c5 100644 --- a/app/projects/controller.py +++ b/app/projects/controller.py @@ -104,9 +104,10 @@ def post(self) -> Project: parser = reqparse.RequestParser() parser.add_argument(name="projectName", type=str) parser.add_argument(name="description", type=str) - parser.add_argument(name="showAllTrees", type=bool) - parser.add_argument(name="exerciseMode", type=bool) + parser.add_argument(name="blindAnnotationMode", type=bool) parser.add_argument(name="visibility", type=int) + parser.add_argument(name="config", type=str) + parser.add_argument(name="language", type=str) args = parser.parse_args() # Sanitize the project name to correspond to Grew folders requirements @@ -115,10 +116,11 @@ def post(self) -> Project: new_project_attrs: ProjectInterface = { "project_name": projectName, "description": args.description, - "show_all_trees": args.showAllTrees, - "exercise_mode": args.exerciseMode, + "blind_annotation_mode": args.blindAnnotationMode, "visibility": args.visibility, "freezed": False, + "config": args.config, + "language": args.language } # KK : TODO : put all grew request in a seperated file and add error catching diff --git a/app/projects/interface.py b/app/projects/interface.py index bbefa86..ecff3d4 100644 --- a/app/projects/interface.py +++ b/app/projects/interface.py @@ -8,9 +8,10 @@ class ProjectInterface(TypedDict, total=False): description: str image: Any visibility: int - show_all_trees: bool - exercise_mode: bool + blind_annotation_mode: bool freezed: bool + config: str + language: str class ProjectExtendedInterface(ProjectInterface, total=False): diff --git a/app/projects/model.py b/app/projects/model.py index cf59446..8478a2e 100644 --- a/app/projects/model.py +++ b/app/projects/model.py @@ -17,11 +17,12 @@ class Project(db.Model, BaseM): description = Column(String(256)) image = Column(String(256), nullable=True) visibility = Column(Integer) - show_all_trees = Column(Boolean, default=True) - exercise_mode = Column(Boolean, default=False) + blind_annotation_mode = Column(Boolean, default=False) diff_mode = Column(Boolean, default=False) diff_user_id = Column(String(256), nullable=True) freezed = Column(Boolean, default=False) + config = Column(String(256), nullable=True) + language = Column(String(256), nullable=True) feature = db.relationship("ProjectFeature", cascade="all,delete", backref="projects") meta_feature = db.relationship("ProjectMetaFeature", cascade="all,delete", backref="projects") diff --git a/app/projects/schema.py b/app/projects/schema.py index 3430e6c..37a85ed 100644 --- a/app/projects/schema.py +++ b/app/projects/schema.py @@ -28,9 +28,10 @@ class ProjectSchema(Schema): description = fields.String(attribute="description") image = BlobImageField(attribute="image") visibility = fields.Integer(attribute="visibility") - showAllTrees = fields.Boolean(attribute="show_all_trees") - exerciseMode = fields.Boolean(attribute="exercise_mode") + blindAnnotationMode = fields.Boolean(attribute="blind_annotation_mode") freezed = fields.Boolean(attribute="freezed") + config = fields.String(attribute="config") + language = fields.String(attribute="language") class ProjectExtendedSchema(ProjectSchema): @@ -58,9 +59,10 @@ class ProjectSchemaCamel(Schema): description = fields.String(attribute="description") image = BlobImageField(attribute="image") visibility = fields.Integer(attribute="visibility") - showAllTrees = fields.Boolean(attribute="show_all_trees") - exerciseMode = fields.Boolean(attribute="exercise_mode") + blindAnnotationMode = fields.Boolean(attribute="blind_annotation_mode") freezed = fields.Boolean(attribute="freezed") + config = fields.String(attribute="config") + language = fields.String(attribute="language") diffMode = fields.Boolean(attribute="diff_mode") diffUserId = fields.String(attribute="diff_user_id") diff --git a/app/projects/service_test.py b/app/projects/service_test.py index b91a9b0..a25af9b 100644 --- a/app/projects/service_test.py +++ b/app/projects/service_test.py @@ -9,7 +9,6 @@ from .interface import ProjectExtendedInterface, ProjectInterface from .model import Project, ProjectAccess, ProjectFeature, ProjectMetaFeature, DefaultUserTrees from ..utils.grew_utils import grew_request -from ..samples.model import SampleRole class ProjectService: diff --git a/app/routes.py b/app/routes.py index 96021d8..19fd952 100644 --- a/app/routes.py +++ b/app/routes.py @@ -3,6 +3,7 @@ def register_routes(api, app, root="api"): from app.projects import register_routes as attach_projects from app.samples import register_routes as attach_samples from app.trees import register_routes as attach_trees + from app.tags import register_routes as attach_tags from app.lexicon import register_routes as attach_lexicon from app.constructicon import register_routes as attach_constructicon from app.grew import register_routes as attach_grew @@ -15,6 +16,7 @@ def register_routes(api, app, root="api"): attach_projects(api, app, root) attach_samples(api, app, root) attach_trees(api, app, root) + attach_tags(api, app, root) attach_lexicon(api, app, root) attach_constructicon(api, app, root) attach_grew(api, app, root) diff --git a/app/samples/controller.py b/app/samples/controller.py index 05796e1..e142906 100644 --- a/app/samples/controller.py +++ b/app/samples/controller.py @@ -10,11 +10,9 @@ from app.user.service import UserService from app.utils.grew_utils import GrewService, SampleExportService, grew_request -from .model import SampleRole from .service import ( SampleEvaluationService, - SampleExerciseLevelService, - SampleRoleService, + SampleBlindAnnotationLevelService, SampleUploadService, SampleTokenizeService, ) @@ -44,16 +42,13 @@ def get(self, project_name: str): "treeByUser": grew_sample["tree_by_user"], "roles": {}, } - sample["roles"] = SampleRoleService.get_by_sample_name( + blind_annotation_level = SampleBlindAnnotationLevelService.get_by_sample_name( project.id, grew_sample["name"] ) - sample_exercise_level = SampleExerciseLevelService.get_by_sample_name( - project.id, grew_sample["name"] - ) - if sample_exercise_level: - sample["exerciseLevel"] = sample_exercise_level.exercise_level.code + if blind_annotation_level: + sample["blindAnnotationLevel"] = blind_annotation_level.blind_annotation_level.code else: - sample["exerciseLevel"] = 4 + sample["blindAnnotationLevel"] = 4 processed_samples.append(sample) return processed_samples @@ -106,63 +101,31 @@ def post(self, project_name): SampleTokenizeService.tokenize(text, option, lang, project_name, sample_name, username) LastAccessService.update_last_access_per_user_and_project(current_user.id, project_name, "write") -@api.route("//samples//role") -class SampleRoleResource(Resource): - def post(self, project_name: str, sample_name: str): - parser = reqparse.RequestParser() - parser.add_argument(name="username", type=str) - parser.add_argument(name="targetrole", type=str) - parser.add_argument(name="action", type=str) - args = parser.parse_args() - - project = ProjectService.get_by_name(project_name) - ProjectAccessService.check_admin_access(project.id) - ProjectService.check_if_freezed(project) - - role = SampleRole.LABEL_TO_ROLES[args.targetrole] - user_id = UserService.get_by_username(args.username).id - if args.action == "add": - new_attrs = { - "project_id": project.id, - "sample_name": sample_name, - "user_id": user_id, - "role": role, - } - SampleRoleService.create(new_attrs) - - if args.action == "remove": - SampleRoleService.delete_one(project.id, sample_name, user_id, role) - - data = { - "roles": SampleRoleService.get_by_sample_name(project.id, sample_name), - } - return data - -@api.route("//samples//exercise-level") -class SampleExerciseLevelResource(Resource): +@api.route("//samples//blind-annotation-level") +class SampleBlindAnnotationLevelResource(Resource): def post(self, project_name: str, sample_name: str): parser = reqparse.RequestParser() - parser.add_argument(name="exerciseLevel", type=str) + parser.add_argument(name="blindAnnotationLevel", type=str) args = parser.parse_args() project = ProjectService.get_by_name(project_name) ProjectAccessService.check_admin_access(project.id) - sample_exercise_level = SampleExerciseLevelService.get_by_sample_name( + sample_blind_annotation_level = SampleBlindAnnotationLevelService.get_by_sample_name( project.id, sample_name ) new_attrs = { "project_id": project.id, "sample_name": sample_name, - "exercise_level": args.exerciseLevel, + "blind_annotation_level": args.blindAnnotationLevel, } - if sample_exercise_level: - SampleExerciseLevelService.update(sample_exercise_level, new_attrs) + if sample_blind_annotation_level: + SampleBlindAnnotationLevelService.update(sample_blind_annotation_level, new_attrs) else: - SampleExerciseLevelService.create(new_attrs) + SampleBlindAnnotationLevelService.create(new_attrs) return {"status": "success"} from app.shared.service import SharedService @@ -215,8 +178,7 @@ def delete(self, project_name: str, sample_name: str): ProjectService.check_if_freezed(project) ProjectService.check_if_project_exist(project) GrewService.delete_sample(project_name, sample_name) - SampleRoleService.delete_by_sample_name(project.id, sample_name) - SampleExerciseLevelService.delete_by_sample_name(project.id, sample_name) + SampleBlindAnnotationLevelService.delete_by_sample_name(project.id, sample_name) LastAccessService.update_last_access_per_user_and_project(current_user.id, project_name, "write") return { "status": "OK", diff --git a/app/samples/model.py b/app/samples/model.py index cc1ee16..d8c2448 100644 --- a/app/samples/model.py +++ b/app/samples/model.py @@ -3,23 +3,10 @@ from app import db # noqa - -class SampleRole(db.Model): - __tablename__ = "samplerole" - ROLES = [(1, "annotator"), (2, "validator")] - LABEL_TO_ROLES = {v: k for k, v in dict(ROLES).items()} - - id = Column(Integer, primary_key=True) - sample_name = Column(String(256), nullable=False) - project_id = Column(Integer, db.ForeignKey("projects.id")) - user_id = Column(String(256), db.ForeignKey("users.id")) - role = Column(ChoiceType(ROLES, impl=Integer())) - - -class SampleExerciseLevel(db.Model): - __tablename__ = "exerciselevel" - EXERCISE_LEVEL = [ - (1, "teacher_visible"), +class SampleBlindAnnotationLevel(db.Model): + __tablename__ = "blindannotationlevel" + BLIND_ANNOTATION_LEVEL = [ + (1, "validated_visible"), (2, "graphical_feedback"), (3, "numerical_feedback"), (4, "no_feedback"), @@ -27,7 +14,7 @@ class SampleExerciseLevel(db.Model): id = Column(Integer, primary_key=True) sample_name = Column(String(256), nullable=False) project_id = Column(Integer, db.ForeignKey("projects.id")) - exercise_level = Column(ChoiceType(EXERCISE_LEVEL, impl=Integer())) + blind_annotation_level = Column(ChoiceType(BLIND_ANNOTATION_LEVEL, impl=Integer())) def update(self, changes): for key, val in changes.items(): diff --git a/app/samples/service.py b/app/samples/service.py index dd405b5..de5396d 100644 --- a/app/samples/service.py +++ b/app/samples/service.py @@ -11,12 +11,11 @@ from app import db from app.config import Config -from app.user.model import User from app.projects.service import ProjectService from app.utils.grew_utils import GrewService from app.github.service import GithubCommitStatusService, GithubSynchronizationService, GithubWorkflowService -from .model import SampleExerciseLevel, SampleRole +from .model import SampleBlindAnnotationLevel BASE_TREE = "base_tree" @@ -98,120 +97,36 @@ def tokenize(text, option, lang, project_name, sample_name, username): GithubCommitStatusService.create(project_name, sample_name) GithubCommitStatusService.update(project_name, sample_name) - -class SampleRoleService: - @staticmethod - def create(new_attrs): - new_sample_role = SampleRole(**new_attrs) - db.session.add(new_sample_role) - db.session.commit() - return new_sample_role - - @staticmethod - def get_one( - project_id: int, - sample_name: str, - user_id: int, - role: int, - ): - """Get one specific user role """ - role = ( - db.session.query(SampleRole) - .filter(SampleRole.user_id == user_id) - .filter(SampleRole.project_id == project_id) - .filter(SampleRole.sample_name == sample_name) - .filter(SampleRole.role == role) - .first() - ) - - @staticmethod - def delete_one( - project_id: int, - sample_name: str, - user_id: int, - role: int, - ): - """Delete one specific user role """ - role = ( - db.session.query(SampleRole) - .filter(SampleRole.user_id == user_id) - .filter(SampleRole.project_id == project_id) - .filter(SampleRole.sample_name == sample_name) - .filter(SampleRole.role == role) - .first() - ) - if not role: - return [] - db.session.delete(role) - db.session.commit() - return [(project_id, sample_name, user_id, role)] - - @staticmethod - def get_by_sample_name(project_id: int, sample_name: str): - """Get a dict of annotators and validators for a given sample""" - roles = {} - for r, label in SampleRole.ROLES: - role = ( - db.session.query(User, SampleRole) - .filter(User.id == SampleRole.user_id) - .filter(SampleRole.project_id == project_id) - .filter(SampleRole.sample_name == sample_name) - .filter(SampleRole.role == r) - .all() - ) - roles[label] = [{"key": a.username, "value": a.username} for a, b in role] - - return roles - - @staticmethod - def delete_by_sample_name(project_id: int, sample_name: str): - """Delete all access of a sample. Used after a sample deletion was asked by the user - ... perform on grew server.""" - roles = ( - db.session.query(SampleRole) - .filter(SampleRole.project_id == project_id) - .filter(SampleRole.sample_name == sample_name) - .all() - ) - for role in roles: - db.session.delete(role) - db.session.commit() - - return - - # def get_annotators_by_sample_id(project_id: int, sample_id: int) -> List[str]: - # return - -class SampleExerciseLevelService: +class SampleBlindAnnotationLevelService: @staticmethod - def create(new_attrs) -> SampleExerciseLevel: - new_sample_access_level = SampleExerciseLevel(**new_attrs) - db.session.add(new_sample_access_level) + def create(new_attrs) -> SampleBlindAnnotationLevel: + new_blind_annotation_level = SampleBlindAnnotationLevel(**new_attrs) + db.session.add(new_blind_annotation_level) db.session.commit() - return new_sample_access_level + return new_blind_annotation_level @staticmethod - def update(sample_exercise_level: SampleExerciseLevel, changes): - sample_exercise_level.update(changes) + def update(blind_annotation_level: SampleBlindAnnotationLevel, changes): + blind_annotation_level.update(changes) db.session.commit() - return sample_exercise_level + return blind_annotation_level @staticmethod - def get_by_sample_name(project_id: int, sample_name: str) -> SampleExerciseLevel: - sample_exercise_level = SampleExerciseLevel.query.filter_by( + def get_by_sample_name(project_id: int, sample_name: str) -> SampleBlindAnnotationLevel: + blind_annotation_level = SampleBlindAnnotationLevel.query.filter_by( sample_name=sample_name, project_id=project_id ).first() - return sample_exercise_level + return blind_annotation_level @staticmethod def delete_by_sample_name(project_id: int, sample_name: str): """Delete all access of a sample. Used after a sample deletion was asked by the user ... perform on grew server.""" roles = ( - db.session.query(SampleExerciseLevel) - .filter(SampleExerciseLevel.project_id == project_id) - .filter(SampleExerciseLevel.sample_name == sample_name) + db.session.query(SampleBlindAnnotationLevel) + .filter(SampleBlindAnnotationLevel.project_id == project_id) + .filter(SampleBlindAnnotationLevel.sample_name == sample_name) .all() ) for role in roles: @@ -229,12 +144,12 @@ def evaluate_sample(sample_conlls): submitted = {} total = {"UPOS": 0, "DEPREL": 0, "HEAD": 0} for sentence_id, sentence_conlls in sample_conlls.items(): - teacher_conll = sentence_conlls.get("teacher") - if teacher_conll: - teacher_sentence_json = sentenceConllToJson( - teacher_conll + validated_tree_conll = sentence_conlls.get("validated") + if validated_tree_conll: + validated_tree_sentence_json = sentenceConllToJson( + validated_tree_conll ) - teacher_tree = teacher_sentence_json["treeJson"]['nodesJson'] + validated_tree = validated_tree_sentence_json["treeJson"]['nodesJson'] basetree_conll = sentence_conlls.get(BASE_TREE) if basetree_conll: @@ -245,15 +160,15 @@ def evaluate_sample(sample_conlls): else: basetree_tree = {} - for token_id in teacher_tree.keys(): - teacher_token = teacher_tree.get(token_id) - if teacher_token == None: + for token_id in validated_tree.keys(): + validated_tree_token = validated_tree.get(token_id) + if validated_tree_token == None: continue basetree_token = basetree_tree.get(token_id, {}) for label in ["UPOS", "HEAD", "DEPREL"]: if ( - teacher_token[label] != "_" - and basetree_token.get(label) != teacher_token[label] + validated_tree_token[label] != "_" + and basetree_token.get(label) != validated_tree_token[label] ): total[label] += 1 else: @@ -261,7 +176,7 @@ def evaluate_sample(sample_conlls): for user_id, user_conll in sentence_conlls.items(): - if user_id != "teacher": + if user_id != "validated": if not corrects.get(user_id): corrects[user_id] = {"UPOS": 0, "DEPREL": 0, "HEAD": 0} if not submitted.get(user_id): @@ -273,8 +188,8 @@ def evaluate_sample(sample_conlls): user_tree = user_sentence_json["treeJson"]["nodesJson"] for token_id in user_tree.keys(): - teacher_token = teacher_tree.get(token_id) - if teacher_token == None: + validated_tree_token = validated_tree.get(token_id) + if validated_tree_token == None: continue user_token = user_tree.get(token_id) @@ -282,13 +197,13 @@ def evaluate_sample(sample_conlls): for label in ["UPOS", "HEAD", "DEPREL"]: if ( - teacher_token[label] != "_" - and basetree_token.get(label) != teacher_token[label] + validated_tree_token[label] != "_" + and basetree_token.get(label) != validated_tree_token[label] ): if user_token[label] != "_": submitted[user_id][label] += 1 corrects[user_id][label] += ( - teacher_token[label] == user_token[label] + validated_tree_token[label] == user_token[label] ) GRADE = 100 evaluations = {} diff --git a/app/tags/__init__.py b/app/tags/__init__.py new file mode 100644 index 0000000..8a51978 --- /dev/null +++ b/app/tags/__init__.py @@ -0,0 +1,7 @@ +BASE_ROUTE = "projects" + + +def register_routes(api, app, root="api"): + from .controller import api as tags_api + + api.add_namespace(tags_api, path=f"/{root}/{BASE_ROUTE}") \ No newline at end of file diff --git a/app/tags/controller.py b/app/tags/controller.py new file mode 100644 index 0000000..889ade6 --- /dev/null +++ b/app/tags/controller.py @@ -0,0 +1,61 @@ +import json + +from flask import request +from flask_restx import Namespace, Resource + +from app.user.service import UserService +from app.projects.service import ProjectService +from .service import TagService, UserTagsService + +api = Namespace( + "Tags", + description="Endpoints for dealing with tags", +) # noqa + +@api.route("//samples//tags") +class TagsResource(Resource): + + def post(self, project_name, sample_name): + + data = request.get_json() + tags = data.get("tags") + tree = data.get("tree") + return TagService.add_new_tags(project_name, sample_name, tags, tree) + + def put(self, project_name, sample_name): + + data = request.get_json() + tag = data.get("tag") + tree = data.get("tree") + return TagService.remove_tag(project_name, sample_name, tag, tree) + + +@api.route("//tags/") +class UserTagsResource(Resource): + + def get(self, project_name, username): + + project = ProjectService.get_by_name(project_name) + ProjectService.check_if_project_exist(project) + user_id = UserService.get_by_username(username).id + if UserTagsService.get_by_user_id(user_id): + return UserTagsService.get_by_user_id(user_id).tags + + + def post(self, project_name, username): + + project = ProjectService.get_by_name(project_name) + ProjectService.check_if_project_exist(project) + data = request.get_json() + tags = data.get("tags") + user_id = UserService.get_by_username(username).id + user_tags = { + "user_id": user_id, + "tags": [tags] + } + UserTagsService.create_or_update(user_tags) + + + + + \ No newline at end of file diff --git a/app/tags/model.py b/app/tags/model.py new file mode 100644 index 0000000..9fd8aac --- /dev/null +++ b/app/tags/model.py @@ -0,0 +1,19 @@ +from sqlalchemy import Column, Integer, String +from sqlalchemy.dialects.postgresql import JSONB + +from app import db +from app.shared.model import BaseM + +class UserTags(db.Model, BaseM): + __tablename__ = 'user_tags' + id = Column(Integer, primary_key=True) + user_id = Column(String(256), db.ForeignKey("users.id")) + tags = Column(JSONB, nullable=False) + + def update(self, changes): + for key, val in changes.items(): + setattr(self, key, val) + return self + + + \ No newline at end of file diff --git a/app/tags/service.py b/app/tags/service.py new file mode 100644 index 0000000..6be26a1 --- /dev/null +++ b/app/tags/service.py @@ -0,0 +1,95 @@ +import json + +from flask import abort +from conllup.conllup import sentenceConllToJson, sentenceJsonToConll + +from app import db + +from app.utils.grew_utils import grew_request +from .model import UserTags + +class TagService: + + @staticmethod + def add_new_tags(project_name, sample_name, tags, conll): + + tags_value = '' + new_tags = ', '.join(tags) + + sentence_json = sentenceConllToJson(conll) + if "tags" in sentence_json["metaJson"].keys(): + existing_tags = sentence_json["metaJson"]["tags"] + tags_value = f"{existing_tags}, {new_tags}" + else: + tags_value = new_tags + + sentence_json["metaJson"]["tags"] = tags_value + user_id = sentence_json["metaJson"]["user_id"] + + conll = sentenceJsonToConll(sentence_json) + grew_request('saveGraph', data= { + "project_id": project_name, + "sample_id": sample_name, + "user_id": user_id, + "conll_graph": conll + }) + return sentence_json["metaJson"] + + @staticmethod + def remove_tag(project_name, sample_name, tag, conll): + + sentence_json = sentenceConllToJson(conll) + user_id = sentence_json["metaJson"]["user_id"] + + if "tags" in sentence_json["metaJson"].keys(): + existing_tags = sentence_json["metaJson"]["tags"].split(",") + existing_tags.remove(tag) + tags_value = ', '.join(existing_tags) + + if tags_value: + sentence_json["metaJson"]["tags"] = tags_value + else: + sentence_json["metaJson"].pop("tags") + + conll = sentenceJsonToConll(sentence_json) + grew_request('saveGraph', data={ + "project_id": project_name, + "sample_id": sample_name, + "user_id": user_id, + "conll_graph": conll + }) + return sentence_json["metaJson"] + else: + abort(406, "This sentence doesn't contain tags") + + +class UserTagsService: + + @staticmethod + def get_by_user_id(user_id) -> UserTags: + return UserTags.query.filter(UserTags.user_id == user_id).first() + + @staticmethod + def create_or_update(new_attrs) -> UserTags: + user_tags_entry = UserTagsService.get_by_user_id(new_attrs.get("user_id")) + if user_tags_entry: + existing_tags = user_tags_entry.tags + new_attrs["tags"] = existing_tags + new_attrs.get("tags") + user_tags_entry.update(new_attrs) + else: + user_tags_entry = UserTags(**new_attrs) + db.session.add(user_tags_entry) + + db.session.commit() + return user_tags_entry + + + + + + + + + + + diff --git a/app/trees/controller.py b/app/trees/controller.py index c41d7f8..a4cc876 100644 --- a/app/trees/controller.py +++ b/app/trees/controller.py @@ -5,12 +5,13 @@ from conllup.processing import constructTextFromTreeJson, emptySentenceConllu, changeMetaFieldInSentenceConllu from app.projects.service import LastAccessService, ProjectAccessService, ProjectService -from app.samples.service import SampleExerciseLevelService +from app.samples.service import SampleBlindAnnotationLevelService from app.github.service import GithubCommitStatusService, GithubSynchronizationService from app.utils.grew_utils import grew_request, GrewService +from .service import TreeService BASE_TREE = "base_tree" -TEACHER = "teacher" +VALIDATED = "validated" api = Namespace( "Trees", description="Endpoints for dealing with trees of a sample" @@ -31,9 +32,10 @@ def get(self, project_name: str, sample_name: str): # ProjectAccessService.require_access_level(project.id, 2) ##### exercise mode block ##### - exercise_mode = project.exercise_mode - project_access: int = 0 - exercise_level: int = 4 + blind_annotation_mode = project.blind_annotation_mode + project_access = 0 + blind_annotation_level = 4 + if current_user.is_authenticated: project_access_obj = ProjectAccessService.get_by_user_id( current_user.id, project.id @@ -43,50 +45,34 @@ def get(self, project_name: str, sample_name: str): project_access = project_access_obj.access_level.code if project.visibility == 0 and project_access == 0: - abort( - 403, - "The project is not visible and you don't have the right privileges", - ) + abort(403, "The project is not visible and you don't have the right privileges") - if exercise_mode: - exercise_level_obj = SampleExerciseLevelService.get_by_sample_name( + if blind_annotation_mode: + blind_annotation_level_obj = SampleBlindAnnotationLevelService.get_by_sample_name( project.id, sample_name ) - if exercise_level_obj: - exercise_level = exercise_level_obj.exercise_level.code + if blind_annotation_level_obj: + blind_annotation_level = blind_annotation_level_obj.blind_annotation_level.code - sample_trees = extract_trees_from_sample(grew_sample_trees, sample_name) - sample_trees = add_base_tree(sample_trees) + sample_trees = TreeService.extract_trees_from_sample(grew_sample_trees, sample_name) + sample_trees = TreeService.add_base_tree(sample_trees) username = "anonymous" if current_user.is_authenticated: username = current_user.username if project_access <= 1: - sample_trees = add_user_tree(sample_trees, username) + sample_trees = TreeService.add_user_tree(sample_trees, username) if project_access <= 1: - restricted_users = [BASE_TREE, TEACHER, username] - sample_trees = restrict_trees(sample_trees, restricted_users) - + restricted_users = [BASE_TREE, VALIDATED, username] + sample_trees = TreeService.restrict_trees(sample_trees, restricted_users) + else: - if project.show_all_trees or project.visibility == 2: - sample_trees = samples2trees(grew_sample_trees, sample_name) - else: - validator = 1 - if validator: - sample_trees = samples2trees( - grew_sample_trees, - sample_name, - ) - else: - sample_trees = samples2trees_with_restrictions( - grew_sample_trees, - sample_name, - current_user, - ) + sample_trees = TreeService.samples2trees(grew_sample_trees, sample_name) + if current_user.is_authenticated: LastAccessService.update_last_access_per_user_and_project(current_user.id, project_name, "read") - data = {"sample_trees": sample_trees, "exercise_level": exercise_level} + data = {"sample_trees": sample_trees, "blind_annotation_level": blind_annotation_level} return data def post(self, project_name: str, sample_name: str): @@ -105,8 +91,8 @@ def post(self, project_name: str, sample_name: str): abort(400) - if project.exercise_mode == 1 and user_id == TEACHER: - conll = changeMetaFieldInSentenceConllu(conll, "user_id", TEACHER) + if project.blind_annotation_mode == 1 and user_id == VALIDATED: + conll = changeMetaFieldInSentenceConllu(conll, "user_id", VALIDATED) data = { "project_id": project_name, "sample_id": sample_name, @@ -131,160 +117,6 @@ def delete(self, project_name: str, sample_name: str, username: str): LastAccessService.update_last_access_per_user_and_project(current_user.id, project_name, "write") -################################################################################ -################## ####################### -################## Tree functions ####################### -################## ####################### -################################################################################ - - -def samples2trees(samples, sample_name): - """ transforms a list of samples into a trees object """ - trees = {} - for sent_id, users in samples.items(): - for user_id, conll in users.items(): - sentenceJson = sentenceConllToJson(conll) - sentence_text = constructTextFromTreeJson(sentenceJson["treeJson"]) - if sent_id not in trees: - trees[sent_id] = { - "sample_name": sample_name, - "sentence": sentence_text, - "sent_id": sent_id, - "conlls": {}, - "matches": {}, - } - trees[sent_id]["conlls"][user_id] = conll - return trees - - -def extract_trees_from_sample(sample, sample_name): - """ transforms a samples into a trees object """ - trees = {} - for sent_id, users in sample.items(): - for user_id, conll in users.items(): - sentenceJson = sentenceConllToJson(conll) - sentence_text = constructTextFromTreeJson(sentenceJson["treeJson"]) - if sent_id not in trees: - trees[sent_id] = { - "sample_name": sample_name, - "sentence": sentence_text, - "sent_id": sent_id, - "conlls": {}, - "matches": {}, - } - trees[sent_id]["conlls"][user_id] = conll - return trees - - -def add_base_tree(trees): - for sent_id, sent_trees in trees.items(): - sent_conlls = sent_trees["conlls"] - list_users = list(sent_conlls.keys()) - if BASE_TREE not in list_users: - model_user = TEACHER if TEACHER in list_users else list_users[0] - model_tree = sent_conlls[model_user] - empty_conllu = emptySentenceConllu(model_tree) - sent_conlls[BASE_TREE] = empty_conllu - return trees - - -def add_user_tree(trees, username): - for sent_id, sent_trees in trees.items(): - sent_conlls = sent_trees["conlls"] - list_users = list(sent_conlls.keys()) - if username not in list_users: - sent_conlls[username] = sent_conlls[BASE_TREE] - return trees - - -def restrict_trees(trees, restricted_users): - for sent_id, sent_trees in trees.items(): - sent_conlls = sent_trees["conlls"] - for user_id in list(sent_conlls.keys()): - if user_id not in restricted_users: - del sent_conlls[user_id] - return trees -def samples2trees_with_restrictions(samples, sample_name, current_user): - """ transforms a list of samples into a trees object and restrict it to user trees and default tree(s) """ - trees = {} - - default_user_trees_ids = [] - default_usernames = list() - default_usernames = default_user_trees_ids - - if current_user.username not in default_usernames: - default_usernames.append(current_user.username) - for sent_id, users in samples.items(): - filtered_users = { - username: users[username] - for username in default_usernames - if username in users - } - for user_id, conll in filtered_users.items(): - sentenceJson = sentenceConllToJson(conll) - sentence_text = constructTextFromTreeJson(sentenceJson["treeJson"]) - if sent_id not in trees: - trees[sent_id] = { - "sample_name": sample_name, - "sentence": sentence_text, - "sent_id": sent_id, - "conlls": {}, - "matches": {}, - } - trees[sent_id]["conlls"][user_id] = conll - return trees - - -def samples2trees_exercise_mode(trees_on_grew, sample_name, current_user, project_name): - """ transforms a list of samples into a trees object and restrict it to user trees and default tree(s) """ - trees_processed = {} - usernames = ["teacher", current_user.username] - - for sent_id, tree_users in trees_on_grew.items(): - trees_processed[sent_id] = { - "sample_name": sample_name, - "sentence": "", - "sent_id": sent_id, - "conlls": {}, - "matches": {}, - } - for username, conll in tree_users.items(): - if username in usernames: - trees_processed[sent_id]["conlls"][username] = conll - # add the sentence to the dict - # TODO : put this script on frontend and not in backend (add a conllu -> sentence in javascript) - # if tree: - if trees_processed[sent_id]["sentence"] == "": - sentenceJson = sentenceConllToJson(conll) - sentence_text = constructTextFromTreeJson(sentenceJson["treeJson"]) - trees_processed[sent_id]["sentence"] = sentence_text - - ### add the base tree (emptied conllu) ### - empty_conllu = emptySentenceConllu(conll) - base_conllu = changeMetaFieldInSentenceConllu(empty_conllu, "user_id", BASE_TREE) - trees_processed[sent_id]["conlls"][BASE_TREE] = base_conllu - - if current_user.username not in trees_processed[sent_id]["conlls"]: - empty_conllu = emptySentenceConllu(conll) - user_empty_conllu = changeMetaFieldInSentenceConllu( - empty_conllu, "user_id", current_user.username - ) - trees_processed[sent_id]["conlls"][ - current_user.username - ] = user_empty_conllu - return trees_processed - - -def get_user_trees(project_name, sample_name, username): - - user_trees_sent_ids = [] - grew_sample_trees = GrewService.get_sample_trees(project_name, sample_name) - sample_trees = extract_trees_from_sample(grew_sample_trees, sample_name) - for sent_id, trees in sample_trees.items(): - if username in trees['conlls']: - user_trees_sent_ids.append(sent_id) - - return user_trees_sent_ids diff --git a/app/trees/service.py b/app/trees/service.py new file mode 100644 index 0000000..3c1e6e4 --- /dev/null +++ b/app/trees/service.py @@ -0,0 +1,120 @@ +from conllup.conllup import sentenceConllToJson +from conllup.processing import constructTextFromTreeJson, emptySentenceConllu, changeMetaFieldInSentenceConllu + +from app.utils.grew_utils import GrewService +BASE_TREE = "base_tree" +VALIDATED = "validated" + +class TreeService: + + @staticmethod + def samples2trees(samples, sample_name): + """ transforms a list of samples into a trees object """ + trees = {} + for sent_id, users in samples.items(): + for user_id, conll in users.items(): + sentence_json = sentenceConllToJson(conll) + sentence_text = constructTextFromTreeJson(sentence_json["treeJson"]) + if sent_id not in trees: + trees[sent_id] = { + "sample_name": sample_name, + "sentence": sentence_text, + "sent_id": sent_id, + "conlls": {}, + "matches": {}, + } + trees[sent_id]["conlls"][user_id] = conll + return trees + + @staticmethod + def extract_trees_from_sample(sample, sample_name): + """ transforms a samples into a trees object """ + trees = {} + for sent_id, users in sample.items(): + for user_id, conll in users.items(): + sentence_json = sentenceConllToJson(conll) + sentence_text = constructTextFromTreeJson(sentence_json["treeJson"]) + if sent_id not in trees: + trees[sent_id] = { + "sample_name": sample_name, + "sentence": sentence_text, + "sent_id": sent_id, + "conlls": {}, + "matches": {}, + } + trees[sent_id]["conlls"][user_id] = conll + return trees + + @staticmethod + def add_base_tree(trees): + for sent_trees in trees.values(): + sent_conlls = sent_trees["conlls"] + list_users = list(sent_conlls.keys()) + if BASE_TREE not in list_users: + model_user = VALIDATED if VALIDATED in list_users else list_users[0] + model_tree = sent_conlls[model_user] + empty_conllu = emptySentenceConllu(model_tree) + sent_conlls[BASE_TREE] = empty_conllu + return trees + + @staticmethod + def add_user_tree(trees, username): + for sent_trees in trees.values(): + sent_conlls = sent_trees["conlls"] + list_users = list(sent_conlls.keys()) + if username not in list_users: + sent_conlls[username] = sent_conlls[BASE_TREE] + return trees + + @staticmethod + def restrict_trees(trees, restricted_users): + for sent_trees in trees.values(): + sent_conlls = sent_trees["conlls"] + for user_id in list(sent_conlls.keys()): + if user_id not in restricted_users: + del sent_conlls[user_id] + return trees + + @staticmethod + def samples2trees_with_restrictions(samples, sample_name, current_user): + """ transforms a list of samples into a trees object and restrict it to user trees and default tree(s) """ + trees = {} + + default_user_trees_ids = [] + default_usernames = list() + default_usernames = default_user_trees_ids + + if current_user.username not in default_usernames: + default_usernames.append(current_user.username) + for sent_id, users in samples.items(): + filtered_users = { + username: users[username] + for username in default_usernames + if username in users + } + for user_id, conll in filtered_users.items(): + sentenceJson = sentenceConllToJson(conll) + sentence_text = constructTextFromTreeJson(sentenceJson["treeJson"]) + if sent_id not in trees: + trees[sent_id] = { + "sample_name": sample_name, + "sentence": sentence_text, + "sent_id": sent_id, + "conlls": {}, + "matches": {}, + } + trees[sent_id]["conlls"][user_id] = conll + return trees + + @staticmethod + def get_user_trees(project_name, sample_name, username): + + user_trees_sent_ids = [] + grew_sample_trees = GrewService.get_sample_trees(project_name, sample_name) + sample_trees = TreeService.extract_trees_from_sample(grew_sample_trees, sample_name) + for sent_id, trees in sample_trees.items(): + if username in trees['conlls']: + user_trees_sent_ids.append(sent_id) + + return user_trees_sent_ids + diff --git a/app/utils/grew_utils.py b/app/utils/grew_utils.py index 2829d20..1956c16 100644 --- a/app/utils/grew_utils.py +++ b/app/utils/grew_utils.py @@ -166,6 +166,8 @@ def search_pattern_in_graphs(project_id: str, pattern: str, user_type: str): user_ids = { "one": [current_user.username, "__last__"] } elif user_type == 'recent': user_ids = { "one": ["__last__"] } + elif user_type == 'validated': + user_ids = { "one": ["validated"] } elif user_type == 'all': user_ids = "all" @@ -187,6 +189,8 @@ def try_package(project_id: str, package: str, sample_ids: List[str] , user_type user_ids = { "one": [current_user.username, "__last__"] } elif user_type == 'recent': user_ids = { "one": ["__last__"] } + elif user_type == 'validated': + user_ids = { "one": ["validated"] } elif user_type == 'all': user_ids = "all" data = { @@ -257,8 +261,24 @@ def get_samples_with_string_contents_as_dict(project_name: str, sample_names: Li else: print("Error: {}".format(reply.get("message"))) return samples_dict_for_user - - + + @staticmethod + def get_validated_trees_filled_up_with_owner_trees(project_name: str, sample_name: str, username: str): + reply = grew_request( + "getConll", + data={"project_id": project_name, "sample_id": sample_name}, + ) + validated_trees = "" + if reply.get("status") == "OK": + sample_tree = SampleExportService.servSampleTrees(reply.get("data", {})) + sample_tree_nots_noui = SampleExportService.servSampleTrees(reply.get("data", {}), timestamps=False, user_ids=False) + for sent_id in sample_tree: + if "validated" in sample_tree[sent_id]["conlls"].keys(): + validated_trees += "".join(sample_tree_nots_noui[sent_id]["conlls"]["validated"]) + else: + validated_trees += "".join(sample_tree_nots_noui[sent_id]["conlls"][username]) + + return validated_trees def get_timestamp(conll): t = re.search(r"# timestamp = (\d+(?:\.\d+)?)\n", conll)