initial commit

This commit is contained in:
joseelinchevalay 2020-10-29 16:37:13 +01:00
commit c9075de3c4
17 changed files with 632 additions and 0 deletions

140
.gitignore vendored Normal file
View File

@ -0,0 +1,140 @@
# Created by https://www.toptal.com/developers/gitignore/api/python
# Edit at https://www.toptal.com/developers/gitignore?templates=python
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
pytestdebug.log
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
doc/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# End of https://www.toptal.com/developers/gitignore/api/python

39
Dockerfile Normal file
View File

@ -0,0 +1,39 @@
FROM debian:buster
LABEL version 1.0+beta.2
ENV JENKINS_HOME /var/jenkins
ENV DEBIAN_FRONTEND noninteractive
ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
#download through the french mirror
RUN sed -i 's/deb\./ftp.fr./' /etc/apt/sources.list
RUN echo "moonlight:/share/dev-common/Applications/x86-64/linux /mnt/applis nfs defaults 0 0" >> /etc/fstab && \
echo "moonlight:/share/home /home nfs defaults 0 0" >> /etc/fstab && \
echo "sharing:/mnt/samba/share /mnt/share nfs defaults 0 0" >> /etc/fstab
# Global config
# FIXME: nfs mounting hangs forever, so no path, etc...
RUN echo "nslcd nslcd/ldap-base string dc=openldap,dc=ullink,dc=lan" | debconf-set-selections && \
echo "nslcd nslcd/ldap-uris string ldap://ldap" | debconf-set-selections && \
echo "libnss-ldapd:amd64 libnss-ldapd/nsswitch multiselect group, passwd, shadow" | debconf-set-selections
RUN apt-get upgrade -y && apt-get update
RUN apt-get -y install \
git \
libnss-ldapd \
libpam-ldapd \
locales \
maven \
nfs-common \
ntp \
openjdk-8-jdk \
openssh-server \
python2.7 \
sudo \
supervisor \
unzip \
vim \
wget \
ca-certificates \
nginx \

79
README.md Normal file
View File

@ -0,0 +1,79 @@
# Mazash
![Swagger UI API documentation](swagger_ui.png)
Minimal application IPFS & Dejavu
The following features are included in the application:
* Recognize song by CID ipfs and extension
* Fingerpint CID IPFS, extension, song name
* API documentation using the OpenAPI 3 specification and Swagger UI
## Setup
To set up the application, you need Python 3. After cloning the repository change to the project directory and install the dependencies via:
```
python3 -m pip install requirements.txt
```
## Development
To start the app in development mode, execute
```
./run_app_dev.sh
```
The application will then be available at `localhost:5000`. You can test the functionality manually using `curl`, e.g. via
```
curl localhost:5000/api/v1/path_for_blueprint_x/test
```
or through the automated tests, by running
```
pytest
```
The commands should output
```
{
"msg": "I'm the test endpoint from blueprint_x."
}
```
and
```
test/test_endpoints.py ....
============= 4 passed in 0.14s ==============
```
respectively.
To view the API documentation through the Swagger user interface, navivate your browser to `localhost:5000/api/docs`.
## Production
To run the app in production, execute
```
./run_app_prod.sh
```
Now the application is served on `localhost:8600`. To run the automated tests for the production host, use
```
pytest --host http://localhost:8600
```
## mp3 for test
https://github.com/worldveil/dejavu/blob/master/mp3/Brad-Sucks--Total-Breakdown.mp3 -> QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r
curl -X POST "http://localhost:8600/api/v1/mazash/recognize" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cid\":\"QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r\",\"extension\":\".mp3\",\"song\":\"Brad-Sucks--Total-Breakdown\"}"
curl -X POST "http://localhost:8600/api/v1/mazash/recognize" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cid\":\"QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r\",\"extension\":\".mp3\"}"

9
app.ini Normal file
View File

@ -0,0 +1,9 @@
[uwsgi]
module = wsgi:app
master = true
processes = 5
http-socket = 0.0.0.0:8600
socket = /tmp/app_socket.sock
chmod-socket = 660
vacuum = true
die-on-term = true

50
docker-compose.yaml Normal file
View File

@ -0,0 +1,50 @@
version: '3'
services:
ipfs:
image: ipfs/go-ipfs:release
ports:
- 5001:5001
- 4001:4001
- 8080:8080
volumes:
- data_ipfs:/data/ipfs
networks:
- db_networks
db:
image: postgres:10.7-alpine
environment:
- POSTGRES_DB=dejavu
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=password
- PGDATA=/var/lib/postgresql/data/pgdata
volumes:
- data_postgresql:/var/lib/postgresql/data/pgdata
networks:
- db_networks
python:
build:
context: ./python
entrypoint: bash -c "pip install -r requirements.txt && pip install https://github.com/worldveil/dejavu/zipball/master && /code/run_app_prod.sh"
environment:
- POSTGRES_HOST=mazash_db_1.mazash_db_networks
- POSTGRES_DB=dejavu
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=password
- IPFS_HOST=ipfs
- IPFS_PORT=5001
volumes:
- .:/code
working_dir: /code
ports:
- 5000:5000
- 8600:8600
depends_on:
- db
- ipfs
networks:
- db_networks
networks:
db_networks:
volumes:
data_ipfs:
data_postgresql:

12
python/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM python:3.7
RUN apt-get update -y && apt-get upgrade -y
RUN apt-get install \
gcc nano \
ffmpeg libasound-dev portaudio19-dev libportaudio2 libportaudiocpp0 \
postgresql postgresql-contrib -y && \
useradd -m app
RUN pip install --upgrade pip
RUN pip install numpy scipy matplotlib pydub pyaudio psycopg2 uwsgi
RUN mkdir /code && chown app /code
USER app
WORKDIR /code

18
requirements.txt Normal file
View File

@ -0,0 +1,18 @@
flask
# testing
pytest
requests
# production
uwsgi
# swagger api docs
apispec
apispec_webframeworks
marshmallow
flask_swagger_ui
openapi_spec_validator
#ipfs
ipfsapi

3
run_app_dev.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
FLASK_ENV=development flask run

3
run_app_prod.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
uwsgi --ini app.ini --need-app

67
src/api_spec.py Normal file
View File

@ -0,0 +1,67 @@
"""OpenAPI v3 Specification"""
# apispec via OpenAPI
from apispec import APISpec
from apispec.ext.marshmallow import MarshmallowPlugin
from apispec_webframeworks.flask import FlaskPlugin
from marshmallow import Schema, fields
# Create an APISpec
spec = APISpec(
title="My App",
version="1.0.0",
openapi_version="3.0.2",
plugins=[FlaskPlugin(), MarshmallowPlugin()],
)
# Define schemas
class FingerprintOuputSchema(Schema):
response = fields.String(description="A message.", required=True)
class FingerprintInputSchema(Schema):
cid = fields.String(description="IPFS cid", required=True)
song = fields.String(description="song name", required=True)
extension = fields.String(description="extension of file", required=True)
class RecognizeResultSchema(Schema):
file_sha1 = fields.String(description="hash of file", required=True)
fingerprinted_confidence = fields.Int(description="", required=True)
fingerprinted_hashes_in_db = fields.Int(description="fingerprinted hashes into database", required=True)
hashes_matched_in_input = fields.Int(description="hashes matched into database", required=True)
input_confidence = fields.Int(description="input confidence", required=True)
input_total_hashes = fields.Int(description="input total hashes", required=True)
offset = fields.Int(description="offset", required=True)
offset_seconds = fields.Int(description="seconds offset", required=True)
song_id = fields.Int(description="song id", required=True)
song_name = fields.String(description="song name", required=True)
class RecognizeOuputSchema(Schema):
align_time = fields.Float(description="", required=True)
fingerprint_time = fields.Float(description="Fingerprinted time", required=True)
query_time = fields.Float(description="query time", required=True)
results = fields.List(fields.Nested(RecognizeResultSchema))
total_time = fields.Float(description="total time of execution", required=True)
class RecognizeInputSchema(Schema):
cid = fields.String(description="IPFS cid", required=True)
extension = fields.String(description="extension of file", required=True)
# register schemas with spec
spec.components.schema("FingerprintOutputResponse", schema=FingerprintOuputSchema)
spec.components.schema("FingerprintInputResponse", schema=FingerprintInputSchema)
spec.components.schema("RecognizeOutputResponse", schema=RecognizeOuputSchema)
spec.components.schema("RecognizeInputResponse", schema=RecognizeInputSchema)
#spec.components.schema("RecognizeResult", schema=RecognizeResultSchema)
# add swagger tags that are used for endpoint annotation
tags = [
{'name': 'mazash',
'description': 'Mazash API'
}
]
for tag in tags:
print(f"Adding tag: {tag['name']}")
spec.tag(tag)

38
src/app.py Normal file
View File

@ -0,0 +1,38 @@
"""Flask Application"""
# load libaries
from flask import Flask, jsonify
import sys
# load modules
from src.endpoints.mazash import mazash
from src.endpoints.swagger import swagger_ui_blueprint, SWAGGER_URL
# init Flask app
app = Flask(__name__)
# register blueprints. ensure that all paths are versioned!
app.register_blueprint(mazash, url_prefix="/api/v1/mazash")
from src.api_spec import spec
# register all swagger documented functions here
with app.test_request_context():
for fn_name in app.view_functions:
if fn_name == 'static':
continue
print(f"Loading swagger docs for function: {fn_name}")
view_fn = app.view_functions[fn_name]
spec.path(view=view_fn)
@app.route("/api/swagger.json")
def create_swagger_spec():
"""
Swagger API definition.
"""
return jsonify(spec.to_dict())
app.register_blueprint(swagger_ui_blueprint, url_prefix=SWAGGER_URL)
if __name__ == "__main__":
app.run(host='0.0.0.0', debug=True)

116
src/endpoints/mazash.py Normal file
View File

@ -0,0 +1,116 @@
from flask import Blueprint, jsonify, request
from dejavu import Dejavu
from dejavu.logic.recognizer.file_recognizer import FileRecognizer
import os
import ipfsapi
import tempfile
mazash = Blueprint(name="mazash", import_name=__name__)
config = {
"database" : {
"host" : os.getenv("POSTGRES_HOST", "db"),
"user" : os.getenv("POSTGRES_USER","postgres"),
"password" : os.getenv("POSTGRES_PASSWORD", "changeme"),
"database" : os.getenv("POSTGRES_DB", "dejavu")
},
"database_type": "postgres"
}
def transform_resultToJsonable(r):
jsonable = {
"song_id": r["song_id"],
"song_name": r["song_name"].decode("utf-8"),
"input_total_hashes": r["input_total_hashes"],
"fingerprinted_hashes_in_db": r["fingerprinted_hashes_in_db"],
"hashes_matched_in_input": r["hashes_matched_in_input"],
"input_confidence": r["input_confidence"],
"fingerprinted_confidence": r["fingerprinted_confidence"],
"offset": int(r["offset"]),
"offset_seconds": r["offset_seconds"],
"file_sha1": r["file_sha1"].decode("utf-8")
}
return jsonable
@mazash.route('/fingerprint', methods=['POST'])
def fingerprint():
"""
---
post:
description: fingerprint a file by CID IPFS
requestBody:
required: true
content:
application/json:
schema: FingerprintInputSchema
responses:
'200':
description: call successful
content:
application/json:
schema: FingerprintOuputSchema
tags:
- mazash
"""
ipfs_api = ipfsapi.connect(os.getenv("IPFS_HOST"), int(os.getenv("IPFS_PORT")))
djv = Dejavu(config)
if request.is_json :
ipfs_hash = request.json.get("cid")
song_name = request.json.get("song")
extension = request.json.get("extension")
content = ipfs_api.cat(ipfs_hash)
temp_file = tempfile.NamedTemporaryFile(suffix=extension)
temp_file.write(content)
link_filename = f"/tmp/{song_name}{extension}"
os.link(temp_file.name, link_filename)
print("{} tempfile created for {}".format(temp_file.name, song_name))
temp_file.close()
djv.fingerprint_directory("/tmp/", [extension])
os.remove(link_filename)
output = {"reponse":"success"}
else :
return 400, "waiting JSON valid"
return jsonify(output)
@mazash.route('/recognize', methods=['POST'])
def recognize():
"""
---
post:
description: recognize a song by CID IPFS
requestBody:
required: true
content:
application/json:
schema: RecognizeInputSchema
responses:
'200':
description: call successful
content:
application/json:
schema: RecognizeOuputSchema
tags:
- mazash
"""
ipfs_api = ipfsapi.connect(os.getenv("IPFS_HOST"), int(os.getenv("IPFS_PORT")))
djv = Dejavu(config)
if request.is_json :
ipfs_hash = request.json.get("cid")
extension = request.json.get("extension")
content = ipfs_api.cat(ipfs_hash)
temp_file = tempfile.NamedTemporaryFile(suffix=extension)
temp_file.write(content)
print("{} tempfile created for {}".format(temp_file.name, ipfs_hash))
song = djv.recognize(FileRecognizer, temp_file.name)
temp_file.close()
print(f"{song}")
result = {
"total_time": song["total_time"],
"fingerprint_time": song["fingerprint_time"],
"query_time": song["query_time"],
"align_time": song["align_time"],
"results": list(map(transform_resultToJsonable, song["results"]))
}
else :
return 400, "waiting JSON valid"
return jsonify(result)

15
src/endpoints/swagger.py Normal file
View File

@ -0,0 +1,15 @@
"""Definition of the Swagger UI Blueprint."""
from flask_swagger_ui import get_swaggerui_blueprint
SWAGGER_URL = '/api/docs'
API_URL = "/api/swagger.json"
# Call factory function to create our blueprint
swagger_ui_blueprint = get_swaggerui_blueprint(
SWAGGER_URL,
API_URL,
config={
'app_name': "My App"
}
)

BIN
swagger_ui.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 88 KiB

16
test/conftest.py Normal file
View File

@ -0,0 +1,16 @@
import pytest
import os
def pytest_addoption(parser):
# ability to test API on different hosts
parser.addoption("--host", action="store", default="http://localhost:5000")
@pytest.fixture(scope="session")
def host(request):
return request.config.getoption("--host")
@pytest.fixture(scope="session")
def api_v1_host(host):
return os.path.join(host, "api", "v1")

14
test/test_endpoints.py Normal file
View File

@ -0,0 +1,14 @@
import os
import requests
from openapi_spec_validator import validate_spec_url
def test_mazash_recognize_test(api_v1_host):
endpoint = os.path.join(api_v1_host, 'mazash', 'count')
response = requests.get(endpoint)
assert response.status_code == 200
def test_swagger_specification(host):
endpoint = os.path.join(host, 'api', 'swagger.json')
validate_spec_url(endpoint)
# use https://editor.swagger.io/ to fix issues

13
wsgi.py Normal file
View File

@ -0,0 +1,13 @@
"""Web Server Gateway Interface"""
##################
# FOR PRODUCTION
####################
from src.app import app
if __name__ == "__main__":
####################
# FOR DEVELOPMENT
####################
app.run(host='0.0.0.0', debug=True)