From 85aef76fed787199f28d7a3dbbb6856cd8e9ae47 Mon Sep 17 00:00:00 2001
From: dig <dig@p2p.legal>
Date: Mon, 22 Apr 2019 21:54:54 +0200
Subject: [PATCH] Basic cryptocompare provider

---
 config.js                  |  8 +++++
 cryptoSaver.js             | 84 ++++++++++++++++++++++++++++++++++++++++++++++
 esm.js                     | 32 ++++++++++++++++++
 package.json               | 21 ++++++++++++
 providers/cryptocompare.js | 31 +++++++++++++++++
 5 files changed, 176 insertions(+)
 create mode 100644 config.js
 create mode 100644 cryptoSaver.js
 create mode 100644 esm.js
 create mode 100644 package.json
 create mode 100644 providers/cryptocompare.js

diff --git a/config.js b/config.js
new file mode 100644
index 0000000..fff505e
--- /dev/null
+++ b/config.js
@@ -0,0 +1,8 @@
+export default {
+	dataDir: './data'
+,	providers: {
+		cryptocompare: {
+			apiKey: ''
+		}
+	}
+}
diff --git a/cryptoSaver.js b/cryptoSaver.js
new file mode 100644
index 0000000..16e5f92
--- /dev/null
+++ b/cryptoSaver.js
@@ -0,0 +1,84 @@
+import 'colors'
+import cron from 'node-schedule'
+import { loadByPath } from './providers/cryptocompare'
+
+//import fetch from 'node-fetch'
+//import cc from 'cryptocompare'
+import fs from 'fs-extra'
+//global.fetch = fetch
+
+import config from './config.js'
+
+const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 }
+const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 }
+
+const json2csv = list=> list.map( p=> [p.time, p.close, p.high, p.low, p.open, p.volumefrom, p.volumeto].join() ).join('\n')
+const csv2json = csv=> csv.split('\n')
+			.map( line=> {
+				let v = line.split(',')
+				return { time: v[0], close: v[1], high: v[2], low: v[3], open: v[4], volumeFrom: v[5], volumeTo:  v[6] }
+			})
+const log = (...args)=> obj=> ( console.log( ...args, obj ), obj )
+
+function ccfs2file( ccfs )
+{
+	let start, end
+//	return loadByPath( ccfs )
+	return fs.readFile( `${config.dataDir}/${ccfs}-1279324800-1555545600.csv`, 'utf8' ).then( csv2json )
+		.then( data=> (start = data[0].time, end = data[data.length-1].time, data) )
+		.then( json2csv )
+		.then( csv=> {
+			let file = `${config.dataDir}/${ccfs}-${start}-${end}.csv`
+//			fs.outputFile( file, csv )
+			return file
+		})
+}
+
+/*
+ccfs2file( 'CCCAGG/BTC/USD/1' )
+ccfs2file( 'CCCAGG/BTC/USD/15' )
+ccfs2file( 'CCCAGG/BTC/USD/1H' )
+ccfs2file( 'CCCAGG/BTC/USD/4H' )
+ccfs2file( 'CCCAGG/BTC/USD/1D' )
+ccfs2file( 'CCCAGG/BTC/USD/1W' )
+ccfs2file( 'CCCAGG/BTC/USD/1M' )
+*/
+
+//cc.rateLimit().then( console.log )
+
+export default function main( ccfs )
+{
+	var job = cron.scheduleJob('*/1 * * * *', ()=>  console.log('The answer to life, the universe, and everything! at', new Date) )
+}
+
+
+export function backfill({ 0:ccfs })
+{
+	let ccPath = ccfs.split('/')
+	,	[ exchange, fsym, tsym, period, start, end = '' ] = ccPath
+
+	if( ccPath.length == 3 ) // all timeFrames
+	{
+		console.log( `Backfilling of all time frames of ${fsym.cyan}/${tsym.blue} on ${exchange.grey} exchange...` )
+		'1 5 15 30 1H 2H 4H 1D 2D 1W 1M'.split(' ')
+			.map( tf=> ccfs+'/'+tf )
+			.map( ccfs=> backfill([ccfs]) )
+	}
+	if( ccPath.length == 4 ) // one timeFrame, recursive until error
+	{
+		console.log( `Backfilling the ${period.magenta} time frame of ${fsym.cyan}/${tsym.blue} on ${exchange.grey} exchange...` )
+		ccfs2file( ccfs )
+			.then( log('File saved:') )
+			.then( file=> log(ccfs+'/'+file.split('-')[1])()|| backfill([ ccfs+'/'+file.split('-')[1] ]) )
+	}
+        if( ccPath.length > 4 ) // from date to date
+        {
+		console.log( `Backfilling from ${start.yellow} to ${end.yellow} of the ${period.magenta} time frame of ${fsym.cyan}/${tsym.blue} on ${exchange.grey} exchange...` )
+        }
+
+}
+
+export function validate({ 0:ccfs, aze })
+{
+	console.log( ccfs, aze )
+}
diff --git a/esm.js b/esm.js
new file mode 100644
index 0000000..e9d57bd
--- /dev/null
+++ b/esm.js
@@ -0,0 +1,32 @@
+let params = process.argv.slice(2)
+
+const moduleStr = params.shift()
+const [ esmodule, exported ] = (moduleStr || '').split(':')
+
+const optionReg = /^-{1,2}(.*?)([:=](.*?))?$/
+const args = params.filter( s=> optionReg.test(s) )
+				.reduce( (opt,str)=> {
+					let [ , prop, , value ] = optionReg.exec(str)
+					return Object.assign( opt, {[prop]: value || true} )
+					// return hasValue ? { ...opt, [prop]: value || true }
+									// : opt
+				}, 
+				params.filter( s=> !optionReg.test(s) )
+			)
+//console.log( process.argv, esmodule, exported, args )
+
+let mod = 
+require('esm')( module, {await: true} )
+( `./${esmodule || require('./package.json').module}` )
+
+let toCall = mod[ exported || 'default' ]
+let returned =
+	typeof toCall == 'function'
+		? toCall( args )
+		: toCall
+//console.log( 'type' , typeof returned, returned instanceof Promise )
+returned instanceof Promise
+	? returned.then( ret=> console.log(ret||''), e=>console.error(e) )
+	: console.log( returned || '' )
+
+module.exports = mod
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..16eab7b
--- /dev/null
+++ b/package.json
@@ -0,0 +1,21 @@
+{
+  "name": "bot",
+  "version": "1.0.0",
+  "description": "",
+  "main": "esm",
+  "module": "index",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "ccxt": "^1.18.60",
+    "colors": "^1.3.3",
+    "cryptocompare": "^1.0.0",
+    "esm": "^3.0.84",
+    "fs-extra": "^7.0.1",
+    "node-fetch": "^2.3.0",
+    "node-schedule": "^1.3.2"
+  }
+}
diff --git a/providers/cryptocompare.js b/providers/cryptocompare.js
new file mode 100644
index 0000000..1161ac0
--- /dev/null
+++ b/providers/cryptocompare.js
@@ -0,0 +1,31 @@
+import fetch from 'node-fetch'
+import cc from 'cryptocompare'
+import fs from 'fs-extra'
+global.fetch = fetch
+
+cc.rateLimit = function( apiKey )
+{
+	return fetch( `https://min-api.cryptocompare.com/stats/rate/limit${apiKey ? `?api_key=${apiKey}` : ''}` )
+		.then( res=> { if( !res.ok ){ throw new Error(`${res.status} ${res.statusText}`) }else return res.json() })
+		.then( body=> { if( body.Response === 'Error' ){ throw body.Message }else return body })
+}
+
+// loadByPath( "EXCHANGE/SYM/SYM/PERIOD/TS|/TS|LIMIT" ).then()
+export function loadByPath( ccfs )
+{
+	let [ exchange, fsym, tsym, period, start, limit = 2000 ] = ccfs.split("/")
+	,       [ , aggregate, tUnit ] = period.match( /(\d+)([HDWM]*)/ )
+	,       loader = 'histo' + (tUnit == '' ? 'Minute' : tUnit == 'H' ? 'Hour' : 'Day')
+	// calculate toTs from limit and start
+	console.log( loader,  aggregate*timeFrames[tUnit] )
+	return cc[loader]( fsym, tsym, { allData: true, limit: 2000, aggregate: aggregate*timeFrames[tUnit], exchange } )
+}
+
+const timeFrames = { '': 1, 'H': 1, 'D': 1, 'W': 7, 'M': 30 }
+const timeFrameMinutes = { '': 1, 'H': 60, 'D': 60*24, 'W': 60*24*7, 'M': 60*24*30 }
+
+
+export {
+	coinList as coins
+,	exchangeList as exchanges
+} from 'cryptocompare'