initial commit
This commit is contained in:
commit
c9075de3c4
|
@ -0,0 +1,140 @@
|
|||
# Created by https://www.toptal.com/developers/gitignore/api/python
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=python
|
||||
|
||||
### Python ###
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
pip-wheel-metadata/
|
||||
share/python-wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
*.py,cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
pytestdebug.log
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
doc/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# IPython
|
||||
profile_default/
|
||||
ipython_config.py
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# pipenv
|
||||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
||||
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
||||
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
||||
# install all needed dependencies.
|
||||
#Pipfile.lock
|
||||
|
||||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
||||
__pypackages__/
|
||||
|
||||
# Celery stuff
|
||||
celerybeat-schedule
|
||||
celerybeat.pid
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
||||
# pytype static type analyzer
|
||||
.pytype/
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/python
|
|
@ -0,0 +1,39 @@
|
|||
FROM debian:buster
|
||||
LABEL version 1.0+beta.2
|
||||
|
||||
ENV JENKINS_HOME /var/jenkins
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
#download through the french mirror
|
||||
RUN sed -i 's/deb\./ftp.fr./' /etc/apt/sources.list
|
||||
|
||||
RUN echo "moonlight:/share/dev-common/Applications/x86-64/linux /mnt/applis nfs defaults 0 0" >> /etc/fstab && \
|
||||
echo "moonlight:/share/home /home nfs defaults 0 0" >> /etc/fstab && \
|
||||
echo "sharing:/mnt/samba/share /mnt/share nfs defaults 0 0" >> /etc/fstab
|
||||
|
||||
# Global config
|
||||
# FIXME: nfs mounting hangs forever, so no path, etc...
|
||||
RUN echo "nslcd nslcd/ldap-base string dc=openldap,dc=ullink,dc=lan" | debconf-set-selections && \
|
||||
echo "nslcd nslcd/ldap-uris string ldap://ldap" | debconf-set-selections && \
|
||||
echo "libnss-ldapd:amd64 libnss-ldapd/nsswitch multiselect group, passwd, shadow" | debconf-set-selections
|
||||
|
||||
RUN apt-get upgrade -y && apt-get update
|
||||
RUN apt-get -y install \
|
||||
git \
|
||||
libnss-ldapd \
|
||||
libpam-ldapd \
|
||||
locales \
|
||||
maven \
|
||||
nfs-common \
|
||||
ntp \
|
||||
openjdk-8-jdk \
|
||||
openssh-server \
|
||||
python2.7 \
|
||||
sudo \
|
||||
supervisor \
|
||||
unzip \
|
||||
vim \
|
||||
wget \
|
||||
ca-certificates \
|
||||
nginx \
|
|
@ -0,0 +1,79 @@
|
|||
# Mazash
|
||||
![Swagger UI API documentation](swagger_ui.png)
|
||||
Minimal application IPFS & Dejavu
|
||||
|
||||
The following features are included in the application:
|
||||
|
||||
* Recognize song by CID ipfs and extension
|
||||
* Fingerpint CID IPFS, extension, song name
|
||||
* API documentation using the OpenAPI 3 specification and Swagger UI
|
||||
|
||||
|
||||
## Setup
|
||||
|
||||
To set up the application, you need Python 3. After cloning the repository change to the project directory and install the dependencies via:
|
||||
|
||||
```
|
||||
python3 -m pip install requirements.txt
|
||||
```
|
||||
|
||||
## Development
|
||||
|
||||
To start the app in development mode, execute
|
||||
|
||||
```
|
||||
./run_app_dev.sh
|
||||
```
|
||||
|
||||
The application will then be available at `localhost:5000`. You can test the functionality manually using `curl`, e.g. via
|
||||
|
||||
```
|
||||
curl localhost:5000/api/v1/path_for_blueprint_x/test
|
||||
```
|
||||
|
||||
or through the automated tests, by running
|
||||
|
||||
```
|
||||
pytest
|
||||
```
|
||||
|
||||
The commands should output
|
||||
```
|
||||
{
|
||||
"msg": "I'm the test endpoint from blueprint_x."
|
||||
}
|
||||
```
|
||||
|
||||
and
|
||||
|
||||
```
|
||||
test/test_endpoints.py ....
|
||||
|
||||
============= 4 passed in 0.14s ==============
|
||||
```
|
||||
|
||||
respectively.
|
||||
|
||||
To view the API documentation through the Swagger user interface, navivate your browser to `localhost:5000/api/docs`.
|
||||
|
||||
## Production
|
||||
|
||||
To run the app in production, execute
|
||||
```
|
||||
./run_app_prod.sh
|
||||
```
|
||||
|
||||
Now the application is served on `localhost:8600`. To run the automated tests for the production host, use
|
||||
|
||||
```
|
||||
pytest --host http://localhost:8600
|
||||
```
|
||||
|
||||
## mp3 for test
|
||||
|
||||
https://github.com/worldveil/dejavu/blob/master/mp3/Brad-Sucks--Total-Breakdown.mp3 -> QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r
|
||||
|
||||
curl -X POST "http://localhost:8600/api/v1/mazash/recognize" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cid\":\"QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r\",\"extension\":\".mp3\",\"song\":\"Brad-Sucks--Total-Breakdown\"}"
|
||||
|
||||
|
||||
curl -X POST "http://localhost:8600/api/v1/mazash/recognize" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"cid\":\"QmU3XRYZiebdDMcUwKrvecxyDgtgVY6zaNYrzQBeCkFb2r\",\"extension\":\".mp3\"}"
|
|
@ -0,0 +1,9 @@
|
|||
[uwsgi]
|
||||
module = wsgi:app
|
||||
master = true
|
||||
processes = 5
|
||||
http-socket = 0.0.0.0:8600
|
||||
socket = /tmp/app_socket.sock
|
||||
chmod-socket = 660
|
||||
vacuum = true
|
||||
die-on-term = true
|
|
@ -0,0 +1,50 @@
|
|||
version: '3'
|
||||
services:
|
||||
ipfs:
|
||||
image: ipfs/go-ipfs:release
|
||||
ports:
|
||||
- 5001:5001
|
||||
- 4001:4001
|
||||
- 8080:8080
|
||||
volumes:
|
||||
- data_ipfs:/data/ipfs
|
||||
networks:
|
||||
- db_networks
|
||||
db:
|
||||
image: postgres:10.7-alpine
|
||||
environment:
|
||||
- POSTGRES_DB=dejavu
|
||||
- POSTGRES_USER=postgres
|
||||
- POSTGRES_PASSWORD=password
|
||||
- PGDATA=/var/lib/postgresql/data/pgdata
|
||||
volumes:
|
||||
- data_postgresql:/var/lib/postgresql/data/pgdata
|
||||
networks:
|
||||
- db_networks
|
||||
python:
|
||||
build:
|
||||
context: ./python
|
||||
entrypoint: bash -c "pip install -r requirements.txt && pip install https://github.com/worldveil/dejavu/zipball/master && /code/run_app_prod.sh"
|
||||
environment:
|
||||
- POSTGRES_HOST=mazash_db_1.mazash_db_networks
|
||||
- POSTGRES_DB=dejavu
|
||||
- POSTGRES_USER=postgres
|
||||
- POSTGRES_PASSWORD=password
|
||||
- IPFS_HOST=ipfs
|
||||
- IPFS_PORT=5001
|
||||
volumes:
|
||||
- .:/code
|
||||
working_dir: /code
|
||||
ports:
|
||||
- 5000:5000
|
||||
- 8600:8600
|
||||
depends_on:
|
||||
- db
|
||||
- ipfs
|
||||
networks:
|
||||
- db_networks
|
||||
networks:
|
||||
db_networks:
|
||||
volumes:
|
||||
data_ipfs:
|
||||
data_postgresql:
|
|
@ -0,0 +1,12 @@
|
|||
FROM python:3.7
|
||||
RUN apt-get update -y && apt-get upgrade -y
|
||||
RUN apt-get install \
|
||||
gcc nano \
|
||||
ffmpeg libasound-dev portaudio19-dev libportaudio2 libportaudiocpp0 \
|
||||
postgresql postgresql-contrib -y && \
|
||||
useradd -m app
|
||||
RUN pip install --upgrade pip
|
||||
RUN pip install numpy scipy matplotlib pydub pyaudio psycopg2 uwsgi
|
||||
RUN mkdir /code && chown app /code
|
||||
USER app
|
||||
WORKDIR /code
|
|
@ -0,0 +1,18 @@
|
|||
flask
|
||||
|
||||
# testing
|
||||
pytest
|
||||
requests
|
||||
|
||||
# production
|
||||
uwsgi
|
||||
|
||||
# swagger api docs
|
||||
apispec
|
||||
apispec_webframeworks
|
||||
marshmallow
|
||||
flask_swagger_ui
|
||||
openapi_spec_validator
|
||||
|
||||
#ipfs
|
||||
ipfsapi
|
|
@ -0,0 +1,3 @@
|
|||
#!/bin/bash
|
||||
FLASK_ENV=development flask run
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
#!/bin/bash
|
||||
uwsgi --ini app.ini --need-app
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
"""OpenAPI v3 Specification"""
|
||||
|
||||
# apispec via OpenAPI
|
||||
from apispec import APISpec
|
||||
from apispec.ext.marshmallow import MarshmallowPlugin
|
||||
from apispec_webframeworks.flask import FlaskPlugin
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
# Create an APISpec
|
||||
spec = APISpec(
|
||||
title="My App",
|
||||
version="1.0.0",
|
||||
openapi_version="3.0.2",
|
||||
plugins=[FlaskPlugin(), MarshmallowPlugin()],
|
||||
)
|
||||
|
||||
# Define schemas
|
||||
class FingerprintOuputSchema(Schema):
|
||||
response = fields.String(description="A message.", required=True)
|
||||
|
||||
class FingerprintInputSchema(Schema):
|
||||
cid = fields.String(description="IPFS cid", required=True)
|
||||
song = fields.String(description="song name", required=True)
|
||||
extension = fields.String(description="extension of file", required=True)
|
||||
|
||||
class RecognizeResultSchema(Schema):
|
||||
file_sha1 = fields.String(description="hash of file", required=True)
|
||||
fingerprinted_confidence = fields.Int(description="", required=True)
|
||||
fingerprinted_hashes_in_db = fields.Int(description="fingerprinted hashes into database", required=True)
|
||||
hashes_matched_in_input = fields.Int(description="hashes matched into database", required=True)
|
||||
input_confidence = fields.Int(description="input confidence", required=True)
|
||||
input_total_hashes = fields.Int(description="input total hashes", required=True)
|
||||
offset = fields.Int(description="offset", required=True)
|
||||
offset_seconds = fields.Int(description="seconds offset", required=True)
|
||||
song_id = fields.Int(description="song id", required=True)
|
||||
song_name = fields.String(description="song name", required=True)
|
||||
|
||||
|
||||
class RecognizeOuputSchema(Schema):
|
||||
align_time = fields.Float(description="", required=True)
|
||||
fingerprint_time = fields.Float(description="Fingerprinted time", required=True)
|
||||
query_time = fields.Float(description="query time", required=True)
|
||||
results = fields.List(fields.Nested(RecognizeResultSchema))
|
||||
total_time = fields.Float(description="total time of execution", required=True)
|
||||
|
||||
class RecognizeInputSchema(Schema):
|
||||
cid = fields.String(description="IPFS cid", required=True)
|
||||
extension = fields.String(description="extension of file", required=True)
|
||||
|
||||
# register schemas with spec
|
||||
spec.components.schema("FingerprintOutputResponse", schema=FingerprintOuputSchema)
|
||||
spec.components.schema("FingerprintInputResponse", schema=FingerprintInputSchema)
|
||||
spec.components.schema("RecognizeOutputResponse", schema=RecognizeOuputSchema)
|
||||
spec.components.schema("RecognizeInputResponse", schema=RecognizeInputSchema)
|
||||
#spec.components.schema("RecognizeResult", schema=RecognizeResultSchema)
|
||||
|
||||
|
||||
# add swagger tags that are used for endpoint annotation
|
||||
tags = [
|
||||
{'name': 'mazash',
|
||||
'description': 'Mazash API'
|
||||
}
|
||||
]
|
||||
|
||||
for tag in tags:
|
||||
print(f"Adding tag: {tag['name']}")
|
||||
spec.tag(tag)
|
|
@ -0,0 +1,38 @@
|
|||
"""Flask Application"""
|
||||
|
||||
# load libaries
|
||||
from flask import Flask, jsonify
|
||||
import sys
|
||||
|
||||
# load modules
|
||||
from src.endpoints.mazash import mazash
|
||||
from src.endpoints.swagger import swagger_ui_blueprint, SWAGGER_URL
|
||||
|
||||
# init Flask app
|
||||
app = Flask(__name__)
|
||||
|
||||
# register blueprints. ensure that all paths are versioned!
|
||||
app.register_blueprint(mazash, url_prefix="/api/v1/mazash")
|
||||
|
||||
from src.api_spec import spec
|
||||
# register all swagger documented functions here
|
||||
|
||||
with app.test_request_context():
|
||||
for fn_name in app.view_functions:
|
||||
if fn_name == 'static':
|
||||
continue
|
||||
print(f"Loading swagger docs for function: {fn_name}")
|
||||
view_fn = app.view_functions[fn_name]
|
||||
spec.path(view=view_fn)
|
||||
|
||||
@app.route("/api/swagger.json")
|
||||
def create_swagger_spec():
|
||||
"""
|
||||
Swagger API definition.
|
||||
"""
|
||||
return jsonify(spec.to_dict())
|
||||
|
||||
app.register_blueprint(swagger_ui_blueprint, url_prefix=SWAGGER_URL)
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host='0.0.0.0', debug=True)
|
|
@ -0,0 +1,116 @@
|
|||
from flask import Blueprint, jsonify, request
|
||||
from dejavu import Dejavu
|
||||
from dejavu.logic.recognizer.file_recognizer import FileRecognizer
|
||||
import os
|
||||
import ipfsapi
|
||||
import tempfile
|
||||
mazash = Blueprint(name="mazash", import_name=__name__)
|
||||
|
||||
config = {
|
||||
"database" : {
|
||||
"host" : os.getenv("POSTGRES_HOST", "db"),
|
||||
"user" : os.getenv("POSTGRES_USER","postgres"),
|
||||
"password" : os.getenv("POSTGRES_PASSWORD", "changeme"),
|
||||
"database" : os.getenv("POSTGRES_DB", "dejavu")
|
||||
},
|
||||
"database_type": "postgres"
|
||||
}
|
||||
|
||||
def transform_resultToJsonable(r):
|
||||
jsonable = {
|
||||
"song_id": r["song_id"],
|
||||
"song_name": r["song_name"].decode("utf-8"),
|
||||
"input_total_hashes": r["input_total_hashes"],
|
||||
"fingerprinted_hashes_in_db": r["fingerprinted_hashes_in_db"],
|
||||
"hashes_matched_in_input": r["hashes_matched_in_input"],
|
||||
"input_confidence": r["input_confidence"],
|
||||
"fingerprinted_confidence": r["fingerprinted_confidence"],
|
||||
"offset": int(r["offset"]),
|
||||
"offset_seconds": r["offset_seconds"],
|
||||
"file_sha1": r["file_sha1"].decode("utf-8")
|
||||
}
|
||||
return jsonable
|
||||
|
||||
@mazash.route('/fingerprint', methods=['POST'])
|
||||
def fingerprint():
|
||||
"""
|
||||
---
|
||||
post:
|
||||
description: fingerprint a file by CID IPFS
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema: FingerprintInputSchema
|
||||
responses:
|
||||
'200':
|
||||
description: call successful
|
||||
content:
|
||||
application/json:
|
||||
schema: FingerprintOuputSchema
|
||||
tags:
|
||||
- mazash
|
||||
"""
|
||||
ipfs_api = ipfsapi.connect(os.getenv("IPFS_HOST"), int(os.getenv("IPFS_PORT")))
|
||||
djv = Dejavu(config)
|
||||
if request.is_json :
|
||||
ipfs_hash = request.json.get("cid")
|
||||
song_name = request.json.get("song")
|
||||
extension = request.json.get("extension")
|
||||
content = ipfs_api.cat(ipfs_hash)
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix=extension)
|
||||
temp_file.write(content)
|
||||
link_filename = f"/tmp/{song_name}{extension}"
|
||||
os.link(temp_file.name, link_filename)
|
||||
print("{} tempfile created for {}".format(temp_file.name, song_name))
|
||||
temp_file.close()
|
||||
djv.fingerprint_directory("/tmp/", [extension])
|
||||
os.remove(link_filename)
|
||||
output = {"reponse":"success"}
|
||||
else :
|
||||
return 400, "waiting JSON valid"
|
||||
return jsonify(output)
|
||||
|
||||
@mazash.route('/recognize', methods=['POST'])
|
||||
def recognize():
|
||||
"""
|
||||
---
|
||||
post:
|
||||
description: recognize a song by CID IPFS
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema: RecognizeInputSchema
|
||||
responses:
|
||||
'200':
|
||||
description: call successful
|
||||
content:
|
||||
application/json:
|
||||
schema: RecognizeOuputSchema
|
||||
tags:
|
||||
- mazash
|
||||
"""
|
||||
ipfs_api = ipfsapi.connect(os.getenv("IPFS_HOST"), int(os.getenv("IPFS_PORT")))
|
||||
djv = Dejavu(config)
|
||||
if request.is_json :
|
||||
ipfs_hash = request.json.get("cid")
|
||||
extension = request.json.get("extension")
|
||||
content = ipfs_api.cat(ipfs_hash)
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix=extension)
|
||||
temp_file.write(content)
|
||||
print("{} tempfile created for {}".format(temp_file.name, ipfs_hash))
|
||||
song = djv.recognize(FileRecognizer, temp_file.name)
|
||||
temp_file.close()
|
||||
print(f"{song}")
|
||||
result = {
|
||||
"total_time": song["total_time"],
|
||||
"fingerprint_time": song["fingerprint_time"],
|
||||
"query_time": song["query_time"],
|
||||
"align_time": song["align_time"],
|
||||
"results": list(map(transform_resultToJsonable, song["results"]))
|
||||
}
|
||||
else :
|
||||
return 400, "waiting JSON valid"
|
||||
|
||||
return jsonify(result)
|
|
@ -0,0 +1,15 @@
|
|||
"""Definition of the Swagger UI Blueprint."""
|
||||
|
||||
from flask_swagger_ui import get_swaggerui_blueprint
|
||||
|
||||
SWAGGER_URL = '/api/docs'
|
||||
API_URL = "/api/swagger.json"
|
||||
|
||||
# Call factory function to create our blueprint
|
||||
swagger_ui_blueprint = get_swaggerui_blueprint(
|
||||
SWAGGER_URL,
|
||||
API_URL,
|
||||
config={
|
||||
'app_name': "My App"
|
||||
}
|
||||
)
|
Binary file not shown.
After Width: | Height: | Size: 88 KiB |
|
@ -0,0 +1,16 @@
|
|||
import pytest
|
||||
import os
|
||||
|
||||
def pytest_addoption(parser):
|
||||
# ability to test API on different hosts
|
||||
parser.addoption("--host", action="store", default="http://localhost:5000")
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def host(request):
|
||||
return request.config.getoption("--host")
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def api_v1_host(host):
|
||||
return os.path.join(host, "api", "v1")
|
||||
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
import os
|
||||
import requests
|
||||
from openapi_spec_validator import validate_spec_url
|
||||
|
||||
def test_mazash_recognize_test(api_v1_host):
|
||||
endpoint = os.path.join(api_v1_host, 'mazash', 'count')
|
||||
response = requests.get(endpoint)
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
def test_swagger_specification(host):
|
||||
endpoint = os.path.join(host, 'api', 'swagger.json')
|
||||
validate_spec_url(endpoint)
|
||||
# use https://editor.swagger.io/ to fix issues
|
Loading…
Reference in New Issue