diff --git a/app/__init__.py b/app/__init__.py index 7e9cadf..074ebd6 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,14 +1,15 @@ -from flask import Flask, jsonify +from flask import Flask, jsonify, send_from_directory from flask_sqlalchemy import SQLAlchemy from flask_restx import Api from flask_login import LoginManager, current_user +from flask_migrate import Migrate from app.klang.config import KlangConfig db = SQLAlchemy() login_manager = LoginManager() - +migrate = Migrate() klang_config = KlangConfig() @@ -31,6 +32,7 @@ def create_app(env=None): register_routes(api, app) db.init_app(app) + migrate.init_app(app, db) login_manager.init_app(app) from .auth import auth as auth_blueprint @@ -41,4 +43,9 @@ def create_app(env=None): def health(): return jsonify("healthy") + ## server for public assets + @app.route('/public/') + def public(path): + return send_from_directory('public', path) + return app diff --git a/app/klang/controller.py b/app/klang/controller.py index 3af3f9c..29db59e 100644 --- a/app/klang/controller.py +++ b/app/klang/controller.py @@ -1,19 +1,26 @@ from datetime import datetime from typing import List -from flask import session +from flask import session, request, abort from flask_accepts.decorators.decorators import responds -from flask_restx import Namespace, Resource +from flask_restx import Namespace, Resource, reqparse +from flask import request, current_app from .service import ConllService +from flask_login import current_user, login_required +import requests api = Namespace("Klang", description="Single namespace, single entity") # noqa + @api.route("/conlls") class ConllServiceResource(Resource): "ConllService" - def get(self) : + def get(self): + # check if the user is logged in + if not current_user.is_authenticated: + return current_app.login_manager.unauthorized() return ConllService.get_all() @@ -22,11 +29,32 @@ class ConllNameServiceResource(Resource): "ConllService" def get(self, conll_name): - + # check if the user is logged in + if not current_user.is_authenticated: + return current_app.login_manager.unauthorized() conll_string = ConllService.get_by_name(conll_name) sentences_string = ConllService.seperate_conll_sentences(conll_string) sentences_audio_token = [] + is_admin = request.args.get('is_admin') + users = ConllService.get_users_list(is_admin) + response = {} + for sentence_string in sentences_string: - audio_tokens = ConllService.sentence_to_audio_tokens(sentence_string) - sentences_audio_token.append(audio_tokens) - return sentences_audio_token \ No newline at end of file + audio_tokens = ConllService.sentence_to_audio_tokens( + sentence_string) + sentences_audio_token.append(audio_tokens) + response['original'] = sentences_audio_token + for user in users: + transcription = ConllService.get_transcription( + user, conll_name, sentences_audio_token) + response[user] = transcription + + return response + + def post(self, conll_name): + data = request.get_json() + transcription = data['transcription'] + if not transcription: + abort(400) + ConllService.save_transcription(conll_name, transcription) + return {'transcription': transcription} \ No newline at end of file diff --git a/app/klang/data_test/John_Doe/John_Doe.intervals.conll b/app/klang/data_test/John_Doe/John_Doe.intervals.conll deleted file mode 100644 index b56309e..0000000 --- a/app/klang/data_test/John_Doe/John_Doe.intervals.conll +++ /dev/null @@ -1,5 +0,0 @@ -# sent_id = John_Doe.intervals.conll__1 -# text = it is -# sound_url = John_Doe.wav -1 it it _ _ _ _ _ _ AlignBegin=100|AlignEnd=500 -2 is is _ _ _ _ _ _ AlignBegin=600|AlignEnd=1000 diff --git a/app/klang/model.py b/app/klang/model.py new file mode 100644 index 0000000..e106d75 --- /dev/null +++ b/app/klang/model.py @@ -0,0 +1,16 @@ +from sqlalchemy import BLOB, Boolean, Column, Integer, String, Boolean, TEXT +from sqlalchemy.ext.declarative import DeclarativeMeta +from sqlalchemy_utils import ChoiceType + +from app import db # noqa + +class Transcription(db.Model): + __tablename__ = "Klang" + # id = models.AutoField(primary_key = True) + # user = models.CharField(max_length = 100) + # mp3 = models.CharField(max_length = 100) + # transcription = models.TextField() + id = Column(Integer, primary_key=True) + user = Column(String(100), unique=True, nullable=False) + mp3 = Column(String(100), unique=True, nullable=False) + transcription = Column(TEXT) diff --git a/app/klang/service.py b/app/klang/service.py index db26a92..f9581bc 100644 --- a/app/klang/service.py +++ b/app/klang/service.py @@ -1,8 +1,15 @@ import os import re from typing import List +from sqlalchemy import exc +import sys +import json +from flask import abort -from app import klang_config +from app import klang_config, db +from .model import Transcription +from app.user.model import User +from flask_login import current_user align_begin_and_end_regex = re.compile( r"^\d+\t(.+?)\t.*AlignBegin=(\d+).*AlignEnd=(\d+)" @@ -46,22 +53,15 @@ def seperate_conll_sentences(conll_string: str) -> List[str]: @staticmethod def sentence_to_audio_tokens(sentence_string: str): - # audio_tokens = {} audio_tokens = [] for line in sentence_string.split("\n"): if line: if not line.startswith("#"): m = align_begin_and_end_regex.search(line) - # audio_token = { - # "token": m.group(1), - # "alignBegin": int(m.group(2)), - # "alignEnd": int(m.group(3)), - # } audio_token = [m.group(1), m.group(2), m.group(3)] audio_tokens.append(audio_token) - # audio_tokens[int(line.split("\t")[0])] = audio_token - print(audio_tokens) + return audio_tokens @staticmethod @@ -73,3 +73,49 @@ def process_sentences_audio_token(conll_name: str): audio_tokens = ConllService.sentence_to_audio_tokens(sentence_string) sentences_audio_token.append(audio_tokens) return sentences_audio_token + + @staticmethod + def get_transcription(user_name, conll_name, original_trans): + trans = [] + try: + record = Transcription.query.filter_by( + user = user_name, + mp3 = conll_name).one() + trans = json.loads(record.transcription) + pass + except exc.SQLAlchemyError: + for line in original_trans: + trans.append([word[0] for word in line]) + pass + return trans + + @staticmethod + def get_users_list(is_admin): + users = [] + if is_admin == 'true': + users = [x.username for x in User.query.all()] + else: + users = [current_user.username] + return users + + @staticmethod + def save_transcription(conll_name, transcription): + user_name = current_user.username + try: + Transcription.query.filter_by( + user = user_name, + mp3 = conll_name + ).delete(synchronize_session = False) + trans_str = json.dumps(transcription) + print(transcription) + record = Transcription( + user = user_name, + mp3 = conll_name, + transcription = trans_str) + db.session.add(record) + db.session.commit() + pass + except: + print(sys.exc_info()[0]) + abort(400) + pass diff --git a/app/public/corpussamples/Thalia_Guevara_Puzma/Thalia_Guevara_Puzma.mp3 b/app/public/corpussamples/Thalia_Guevara_Puzma/Thalia_Guevara_Puzma.mp3 new file mode 100644 index 0000000..b2802f4 Binary files /dev/null and b/app/public/corpussamples/Thalia_Guevara_Puzma/Thalia_Guevara_Puzma.mp3 differ diff --git a/app/public/corpussamples/Veronique_Jezewski/Veronique_Jezewski.mp3 b/app/public/corpussamples/Veronique_Jezewski/Veronique_Jezewski.mp3 new file mode 100644 index 0000000..35ec8d8 Binary files /dev/null and b/app/public/corpussamples/Veronique_Jezewski/Veronique_Jezewski.mp3 differ diff --git a/app/public/corpussamples/Zaynab_Affes/Zaynab_Affes.mp3 b/app/public/corpussamples/Zaynab_Affes/Zaynab_Affes.mp3 new file mode 100644 index 0000000..0ad29a4 Binary files /dev/null and b/app/public/corpussamples/Zaynab_Affes/Zaynab_Affes.mp3 differ diff --git a/app/public/corpussamples/Zoe_Sacotte/Zoe_Sacotte.mp3 b/app/public/corpussamples/Zoe_Sacotte/Zoe_Sacotte.mp3 new file mode 100644 index 0000000..04d7ba2 Binary files /dev/null and b/app/public/corpussamples/Zoe_Sacotte/Zoe_Sacotte.mp3 differ diff --git a/app/public/enregistrement_sonore_Richard_Matthieu.mp3 b/app/public/enregistrement_sonore_Richard_Matthieu.mp3 new file mode 100644 index 0000000..5bbd2f1 Binary files /dev/null and b/app/public/enregistrement_sonore_Richard_Matthieu.mp3 differ diff --git a/app/public/favicon.ico b/app/public/favicon.ico new file mode 100644 index 0000000..7ace6d1 Binary files /dev/null and b/app/public/favicon.ico differ diff --git a/app/public/icons/favicon-128x128.png b/app/public/icons/favicon-128x128.png new file mode 100644 index 0000000..2d6e88c Binary files /dev/null and b/app/public/icons/favicon-128x128.png differ diff --git a/app/public/icons/favicon-16x16.png b/app/public/icons/favicon-16x16.png new file mode 100644 index 0000000..436be16 Binary files /dev/null and b/app/public/icons/favicon-16x16.png differ diff --git a/app/public/icons/favicon-32x32.png b/app/public/icons/favicon-32x32.png new file mode 100644 index 0000000..b605ace Binary files /dev/null and b/app/public/icons/favicon-32x32.png differ diff --git a/app/public/icons/favicon-96x96.png b/app/public/icons/favicon-96x96.png new file mode 100644 index 0000000..3a50144 Binary files /dev/null and b/app/public/icons/favicon-96x96.png differ diff --git a/manage.py b/manage.py index b0d7722..94e1640 100644 --- a/manage.py +++ b/manage.py @@ -4,8 +4,10 @@ from app import create_app, db from commands.seed_command import SeedCommand + from dotenv import load_dotenv load_dotenv(dotenv_path=".flaskenv", verbose=True) +from sqlalchemy import MetaData, Table, Column, Integer, String env = os.getenv("FLASK_ENV") or "test" print(f"Active environment: * {env} *") diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 0000000..f8ed480 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..7645255 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,97 @@ +from __future__ import with_statement + +import logging +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app +print(current_app.extensions['migrate']) +config.set_main_option( + 'sqlalchemy.url', + str(current_app.extensions['migrate'].db.engine.url).replace('%', '%%')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/7d9505c0a5d1_.py b/migrations/versions/7d9505c0a5d1_.py new file mode 100644 index 0000000..7033efb --- /dev/null +++ b/migrations/versions/7d9505c0a5d1_.py @@ -0,0 +1,43 @@ +"""empty message + +Revision ID: 7d9505c0a5d1 +Revises: +Create Date: 2020-11-13 18:30:03.579354 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '7d9505c0a5d1' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('Klang', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user', sa.String(length=100), nullable=False), + sa.Column('mp3', sa.String(length=100), nullable=False), + sa.Column('transcription', sa.TEXT(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('mp3'), + sa.UniqueConstraint('user') + ) + op.drop_table('widget') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('widget', + sa.Column('widget_id', sa.INTEGER(), nullable=False), + sa.Column('name', sa.VARCHAR(length=255), nullable=True), + sa.Column('purpose', sa.VARCHAR(length=255), nullable=True), + sa.PrimaryKeyConstraint('widget_id') + ) + op.drop_table('Klang') + # ### end Alembic commands ###