commit c9075de3c436e41eff69a564a09e51ef67572955 Author: joseelinchevalay Date: Thu Oct 29 16:37:13 2020 +0100 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bf11e46 --- /dev/null +++ b/.gitignore @@ -0,0 +1,140 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# End of https://www.toptal.com/developers/gitignore/api/python \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..384b148 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM debian:buster +LABEL version 1.0+beta.2 + +ENV JENKINS_HOME /var/jenkins +ENV DEBIAN_FRONTEND noninteractive +ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt + +#download through the french mirror +RUN sed -i 's/deb\./ftp.fr./' /etc/apt/sources.list + +RUN echo "moonlight:/share/dev-common/Applications/x86-64/linux /mnt/applis nfs defaults 0 0" >> /etc/fstab && \ + echo "moonlight:/share/home /home nfs defaults 0 0" >> /etc/fstab && \ + echo "sharing:/mnt/samba/share /mnt/share nfs defaults 0 0" >> /etc/fstab + +# Global config +# FIXME: nfs mounting hangs forever, so no path, etc... +RUN echo "nslcd nslcd/ldap-base string dc=openldap,dc=ullink,dc=lan" | debconf-set-selections && \ + echo "nslcd nslcd/ldap-uris string ldap://ldap" | debconf-set-selections && \ + echo "libnss-ldapd:amd64 libnss-ldapd/nsswitch multiselect group, passwd, shadow" | debconf-set-selections + +RUN apt-get upgrade -y && apt-get update +RUN apt-get -y install \ + git \ + libnss-ldapd \ + libpam-ldapd \ + locales \ + maven \ + nfs-common \ + ntp \ + openjdk-8-jdk \ + openssh-server \ + python2.7 \ + sudo \ + supervisor \ + unzip \ + vim \ + wget \ + ca-certificates \ + nginx \ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..cb314d7 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# Mazash +![Swagger UI API documentation](swagger_ui.png) +Minimal application IPFS & Dejavu + +The following features are included in the application: + +* Recognize song by CID ipfs and extension +* Fingerpint CID IPFS, extension, song name +* API documentation using the OpenAPI 3 specification and Swagger UI + + +## Setup + +To set up the application, you need Python 3. After cloning the repository change to the project directory and install the dependencies via: + +``` +python3 -m pip install requirements.txt +``` + +## Development + +To start the app in development mode, execute + +``` +./run_app_dev.sh +``` + +The application will then be available at `localhost:5000`. You can test the functionality manually using `curl`, e.g. via + +``` +curl localhost:5000/api/v1/path_for_blueprint_x/test +``` + +or through the automated tests, by running + +``` +pytest +``` + +The commands should output +``` +{ + "msg": "I'm the test endpoint from blueprint_x." +} +``` + +and + +``` +test/test_endpoints.py .... + +============= 4 passed in 0.14s ============== +``` + +respectively. + +To view the API documentation through the Swagger user interface, navivate your browser to `localhost:5000/api/docs`. + +## Production + +To run the app in production, execute +``` +./run_app_prod.sh +``` + +Now the application is served on `localhost:8600`. To run the automated tests for the production host, use + +``` +pytest --host http://localhost:8600 +``` + +## mp3 for test + +https://github.com/worldveil/dejavu/blob/master/mp3/Brad-Sucks--Total-Breakdown.mp3 -> QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r + +curl -X POST "http://localhost:8600/api/v1/mazash/recognize" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cid\":\"QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r\",\"extension\":\".mp3\",\"song\":\"Brad-Sucks--Total-Breakdown\"}" + + +curl -X POST "http://localhost:8600/api/v1/mazash/recognize" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cid\":\"QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r\",\"extension\":\".mp3\"}" \ No newline at end of file diff --git a/app.ini b/app.ini new file mode 100644 index 0000000..73580ed --- /dev/null +++ b/app.ini @@ -0,0 +1,9 @@ +[uwsgi] +module = wsgi:app +master = true +processes = 5 +http-socket = 0.0.0.0:8600 +socket = /tmp/app_socket.sock +chmod-socket = 660 +vacuum = true +die-on-term = true diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..cb2054f --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,50 @@ +version: '3' +services: + ipfs: + image: ipfs/go-ipfs:release + ports: + - 5001:5001 + - 4001:4001 + - 8080:8080 + volumes: + - data_ipfs:/data/ipfs + networks: + - db_networks + db: + image: postgres:10.7-alpine + environment: + - POSTGRES_DB=dejavu + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=password + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - data_postgresql:/var/lib/postgresql/data/pgdata + networks: + - db_networks + python: + build: + context: ./python + entrypoint: bash -c "pip install -r requirements.txt && pip install https://github.com/worldveil/dejavu/zipball/master && /code/run_app_prod.sh" + environment: + - POSTGRES_HOST=mazash_db_1.mazash_db_networks + - POSTGRES_DB=dejavu + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=password + - IPFS_HOST=ipfs + - IPFS_PORT=5001 + volumes: + - .:/code + working_dir: /code + ports: + - 5000:5000 + - 8600:8600 + depends_on: + - db + - ipfs + networks: + - db_networks +networks: + db_networks: +volumes: + data_ipfs: + data_postgresql: diff --git a/python/Dockerfile b/python/Dockerfile new file mode 100644 index 0000000..4155a6f --- /dev/null +++ b/python/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.7 +RUN apt-get update -y && apt-get upgrade -y +RUN apt-get install \ + gcc nano \ + ffmpeg libasound-dev portaudio19-dev libportaudio2 libportaudiocpp0 \ + postgresql postgresql-contrib -y && \ + useradd -m app +RUN pip install --upgrade pip +RUN pip install numpy scipy matplotlib pydub pyaudio psycopg2 uwsgi +RUN mkdir /code && chown app /code +USER app +WORKDIR /code \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e6c0c48 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,18 @@ +flask + +# testing +pytest +requests + +# production +uwsgi + +# swagger api docs +apispec +apispec_webframeworks +marshmallow +flask_swagger_ui +openapi_spec_validator + +#ipfs +ipfsapi diff --git a/run_app_dev.sh b/run_app_dev.sh new file mode 100755 index 0000000..a5d0b57 --- /dev/null +++ b/run_app_dev.sh @@ -0,0 +1,3 @@ +#!/bin/bash +FLASK_ENV=development flask run + diff --git a/run_app_prod.sh b/run_app_prod.sh new file mode 100755 index 0000000..2da6824 --- /dev/null +++ b/run_app_prod.sh @@ -0,0 +1,3 @@ +#!/bin/bash +uwsgi --ini app.ini --need-app + diff --git a/src/api_spec.py b/src/api_spec.py new file mode 100644 index 0000000..61a0d10 --- /dev/null +++ b/src/api_spec.py @@ -0,0 +1,67 @@ +"""OpenAPI v3 Specification""" + +# apispec via OpenAPI +from apispec import APISpec +from apispec.ext.marshmallow import MarshmallowPlugin +from apispec_webframeworks.flask import FlaskPlugin +from marshmallow import Schema, fields + +# Create an APISpec +spec = APISpec( + title="My App", + version="1.0.0", + openapi_version="3.0.2", + plugins=[FlaskPlugin(), MarshmallowPlugin()], +) + +# Define schemas +class FingerprintOuputSchema(Schema): + response = fields.String(description="A message.", required=True) + +class FingerprintInputSchema(Schema): + cid = fields.String(description="IPFS cid", required=True) + song = fields.String(description="song name", required=True) + extension = fields.String(description="extension of file", required=True) + +class RecognizeResultSchema(Schema): + file_sha1 = fields.String(description="hash of file", required=True) + fingerprinted_confidence = fields.Int(description="", required=True) + fingerprinted_hashes_in_db = fields.Int(description="fingerprinted hashes into database", required=True) + hashes_matched_in_input = fields.Int(description="hashes matched into database", required=True) + input_confidence = fields.Int(description="input confidence", required=True) + input_total_hashes = fields.Int(description="input total hashes", required=True) + offset = fields.Int(description="offset", required=True) + offset_seconds = fields.Int(description="seconds offset", required=True) + song_id = fields.Int(description="song id", required=True) + song_name = fields.String(description="song name", required=True) + + +class RecognizeOuputSchema(Schema): + align_time = fields.Float(description="", required=True) + fingerprint_time = fields.Float(description="Fingerprinted time", required=True) + query_time = fields.Float(description="query time", required=True) + results = fields.List(fields.Nested(RecognizeResultSchema)) + total_time = fields.Float(description="total time of execution", required=True) + +class RecognizeInputSchema(Schema): + cid = fields.String(description="IPFS cid", required=True) + extension = fields.String(description="extension of file", required=True) + +# register schemas with spec +spec.components.schema("FingerprintOutputResponse", schema=FingerprintOuputSchema) +spec.components.schema("FingerprintInputResponse", schema=FingerprintInputSchema) +spec.components.schema("RecognizeOutputResponse", schema=RecognizeOuputSchema) +spec.components.schema("RecognizeInputResponse", schema=RecognizeInputSchema) +#spec.components.schema("RecognizeResult", schema=RecognizeResultSchema) + + +# add swagger tags that are used for endpoint annotation +tags = [ + {'name': 'mazash', + 'description': 'Mazash API' + } + ] + +for tag in tags: + print(f"Adding tag: {tag['name']}") + spec.tag(tag) diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..2ba2815 --- /dev/null +++ b/src/app.py @@ -0,0 +1,38 @@ +"""Flask Application""" + +# load libaries +from flask import Flask, jsonify +import sys + +# load modules +from src.endpoints.mazash import mazash +from src.endpoints.swagger import swagger_ui_blueprint, SWAGGER_URL + +# init Flask app +app = Flask(__name__) + +# register blueprints. ensure that all paths are versioned! +app.register_blueprint(mazash, url_prefix="/api/v1/mazash") + +from src.api_spec import spec +# register all swagger documented functions here + +with app.test_request_context(): + for fn_name in app.view_functions: + if fn_name == 'static': + continue + print(f"Loading swagger docs for function: {fn_name}") + view_fn = app.view_functions[fn_name] + spec.path(view=view_fn) + +@app.route("/api/swagger.json") +def create_swagger_spec(): + """ + Swagger API definition. + """ + return jsonify(spec.to_dict()) + +app.register_blueprint(swagger_ui_blueprint, url_prefix=SWAGGER_URL) + +if __name__ == "__main__": + app.run(host='0.0.0.0', debug=True) diff --git a/src/endpoints/mazash.py b/src/endpoints/mazash.py new file mode 100644 index 0000000..68235ca --- /dev/null +++ b/src/endpoints/mazash.py @@ -0,0 +1,116 @@ +from flask import Blueprint, jsonify, request +from dejavu import Dejavu +from dejavu.logic.recognizer.file_recognizer import FileRecognizer +import os +import ipfsapi +import tempfile +mazash = Blueprint(name="mazash", import_name=__name__) + +config = { + "database" : { + "host" : os.getenv("POSTGRES_HOST", "db"), + "user" : os.getenv("POSTGRES_USER","postgres"), + "password" : os.getenv("POSTGRES_PASSWORD", "changeme"), + "database" : os.getenv("POSTGRES_DB", "dejavu") + }, + "database_type": "postgres" +} + +def transform_resultToJsonable(r): + jsonable = { + "song_id": r["song_id"], + "song_name": r["song_name"].decode("utf-8"), + "input_total_hashes": r["input_total_hashes"], + "fingerprinted_hashes_in_db": r["fingerprinted_hashes_in_db"], + "hashes_matched_in_input": r["hashes_matched_in_input"], + "input_confidence": r["input_confidence"], + "fingerprinted_confidence": r["fingerprinted_confidence"], + "offset": int(r["offset"]), + "offset_seconds": r["offset_seconds"], + "file_sha1": r["file_sha1"].decode("utf-8") + } + return jsonable + +@mazash.route('/fingerprint', methods=['POST']) +def fingerprint(): + """ + --- + post: + description: fingerprint a file by CID IPFS + requestBody: + required: true + content: + application/json: + schema: FingerprintInputSchema + responses: + '200': + description: call successful + content: + application/json: + schema: FingerprintOuputSchema + tags: + - mazash + """ + ipfs_api = ipfsapi.connect(os.getenv("IPFS_HOST"), int(os.getenv("IPFS_PORT"))) + djv = Dejavu(config) + if request.is_json : + ipfs_hash = request.json.get("cid") + song_name = request.json.get("song") + extension = request.json.get("extension") + content = ipfs_api.cat(ipfs_hash) + temp_file = tempfile.NamedTemporaryFile(suffix=extension) + temp_file.write(content) + link_filename = f"/tmp/{song_name}{extension}" + os.link(temp_file.name, link_filename) + print("{} tempfile created for {}".format(temp_file.name, song_name)) + temp_file.close() + djv.fingerprint_directory("/tmp/", [extension]) + os.remove(link_filename) + output = {"reponse":"success"} + else : + return 400, "waiting JSON valid" + return jsonify(output) + +@mazash.route('/recognize', methods=['POST']) +def recognize(): + """ + --- + post: + description: recognize a song by CID IPFS + requestBody: + required: true + content: + application/json: + schema: RecognizeInputSchema + responses: + '200': + description: call successful + content: + application/json: + schema: RecognizeOuputSchema + tags: + - mazash + """ + ipfs_api = ipfsapi.connect(os.getenv("IPFS_HOST"), int(os.getenv("IPFS_PORT"))) + djv = Dejavu(config) + if request.is_json : + ipfs_hash = request.json.get("cid") + extension = request.json.get("extension") + content = ipfs_api.cat(ipfs_hash) + temp_file = tempfile.NamedTemporaryFile(suffix=extension) + temp_file.write(content) + print("{} tempfile created for {}".format(temp_file.name, ipfs_hash)) + song = djv.recognize(FileRecognizer, temp_file.name) + temp_file.close() + print(f"{song}") + result = { + "total_time": song["total_time"], + "fingerprint_time": song["fingerprint_time"], + "query_time": song["query_time"], + "align_time": song["align_time"], + "results": list(map(transform_resultToJsonable, song["results"])) + } + else : + return 400, "waiting JSON valid" + + return jsonify(result) \ No newline at end of file diff --git a/src/endpoints/swagger.py b/src/endpoints/swagger.py new file mode 100644 index 0000000..8d8976a --- /dev/null +++ b/src/endpoints/swagger.py @@ -0,0 +1,15 @@ +"""Definition of the Swagger UI Blueprint.""" + +from flask_swagger_ui import get_swaggerui_blueprint + +SWAGGER_URL = '/api/docs' +API_URL = "/api/swagger.json" + +# Call factory function to create our blueprint +swagger_ui_blueprint = get_swaggerui_blueprint( + SWAGGER_URL, + API_URL, + config={ + 'app_name': "My App" + } +) diff --git a/swagger_ui.png b/swagger_ui.png new file mode 100644 index 0000000..f351fdf Binary files /dev/null and b/swagger_ui.png differ diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..f2e3d36 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,16 @@ +import pytest +import os + +def pytest_addoption(parser): + # ability to test API on different hosts + parser.addoption("--host", action="store", default="http://localhost:5000") + +@pytest.fixture(scope="session") +def host(request): + return request.config.getoption("--host") + +@pytest.fixture(scope="session") +def api_v1_host(host): + return os.path.join(host, "api", "v1") + + diff --git a/test/test_endpoints.py b/test/test_endpoints.py new file mode 100644 index 0000000..979a065 --- /dev/null +++ b/test/test_endpoints.py @@ -0,0 +1,14 @@ +import os +import requests +from openapi_spec_validator import validate_spec_url + +def test_mazash_recognize_test(api_v1_host): + endpoint = os.path.join(api_v1_host, 'mazash', 'count') + response = requests.get(endpoint) + assert response.status_code == 200 + + +def test_swagger_specification(host): + endpoint = os.path.join(host, 'api', 'swagger.json') + validate_spec_url(endpoint) + # use https://editor.swagger.io/ to fix issues diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 0000000..83eb225 --- /dev/null +++ b/wsgi.py @@ -0,0 +1,13 @@ +"""Web Server Gateway Interface""" + +################## +# FOR PRODUCTION +#################### +from src.app import app + +if __name__ == "__main__": + #################### + # FOR DEVELOPMENT + #################### + app.run(host='0.0.0.0', debug=True) +