diff --git a/.circleci/config.yml b/.circleci/config.yml index 971e585..4e84e0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,8 +35,11 @@ jobs: name: Deploy example to aws command: | if [[ "$CIRCLE_BRANCH" == 'master' ]]; then - cd example - yarn - ./node_modules/.bin/kes cf deploy --template node_modules/@sat-utils/api/template --deployment dev --region us-east-1 --role arn:aws:iam::552819999234:role/sat-api-deployer-role-DeployerRole-JBBKUABAELDR + #cd example + #yarn + #./node_modules/.bin/kes cf deploy --template node_modules/@sat-utils/api/template --deployment dev --region us-east-1 --role arn:aws:iam::552819999234:role/sat-api-deployer-role-DeployerRole-JBBKUABAELDR + echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc + VERSION=$(jq --raw-output .version lerna.json) + node_modules/.bin/lerna publish --skip-git --repo-version $VERSION --yes --force-publish=* fi diff --git a/CHANGES.md b/CHANGES.md index 4430b97..cd3f5bf 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v0.1.0] - 2018-09-18 + +### Fixed +- Fixed broken ingests for Landsat and Sentinel + +## [v0.0.2] + ### Added - Added support for [STAC specification](https://github.com/radiantearth/stac-spec/) - The following packages are released @@ -25,19 +32,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - /geojson endpoint - /count endpoint - - -## [v2.0.0] - 2017-06-14 +## [legacy-v2.0.0] - 2018-01-01 - Moves all the metadata indexing logics to the same repo - Uses CloudFormation for full deployment - Includes a better ApiGateway support - -## [v1.0.0] - 2016-10-20 - Use streams to read, transform, and write into elasticsearch - Use batches of lambdas to speed up processing - Refactor several modules: metadata, landsat, sentinel - Refactor and improve splitting -[Unreleased]: https://github.com/cumulus-nasa/cumulus/compare/v2.0.0...HEAD -[v2.0.0]: https://github.com/cumulus-nasa/cumulus/compare/v1.0.0...v2.0.0 \ No newline at end of file +[Unreleased]: https://github.com/sat-utils/sat-api/compare/v0.1.0...HEAD +[v0.1.0]: https://github.com/sat-utils/sat-api/compare/v0.0.2...v0.1.0 +[v0.0.2]: https://github.com/sat-utils/sat-api/compare/legacy-v2.0.0...v0.0.2 +[legacy-v2.0.0]: https://github.com/sat-utils/sat-api/tree/legacy \ No newline at end of file diff --git a/README.md b/README.md index 10149dc..79d27e4 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Sat-api includes a number of NPM packages (in the packages/ directory) that are $ yarn docs-serve -On Linux, if you get the message "Error: watch *path*/book.json ENOSPC", issue the following command (require sudo access). +On Linux, if you get the message "Error: watch *path*/book.json ENOSPC", issue the following command (requires sudo access). $ echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p diff --git a/example/.kes/config.yml b/example/.kes/config.yml index a9d9b9c..cd8eeaa 100644 --- a/example/.kes/config.yml +++ b/example/.kes/config.yml @@ -14,9 +14,9 @@ dev: volumeSize: 80 prod: - stackName: sat-api-v1 + stackName: sat-api system_bucket: sat-api es: instanceCount: 2 instanceType: m3.medium.elasticsearch - volumeSize: 80 \ No newline at end of file + volumeSize: 80 diff --git a/lerna.json b/lerna.json index 9a530a3..543442c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "lerna": "2.11.0", - "version": "0.0.2", + "version": "0.1.0", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 929ee2b..92ffabe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,4 @@ { - "name": "sat-api", - "version": "1.0.0", "description": "One API to search public Satellites metadata", "main": "index.js", "repository": "https://github.com/sat-utils/sat-api", diff --git a/packages/api-lib/libs/api.js b/packages/api-lib/libs/api.js index b249f45..aa51686 100644 --- a/packages/api-lib/libs/api.js +++ b/packages/api-lib/libs/api.js @@ -15,7 +15,7 @@ function Search(event, esClient) { params = event.body } - this.headers = event.headers + this.endpoint = event.endpoint this.merge = false if (_.has(params, 'merge')) { @@ -34,17 +34,13 @@ function Search(event, esClient) { this.client = esClient this.queries = queries(this.params) - - console.log(`Queries: ${JSON.stringify(this.queries)}`) } // search for items using collection and items -Search.prototype.search_items = (callback) => { +Search.prototype.search_items = function (callback) { // check collection first this.search_collections((err, resp) => { const collections = resp.features.map((c) => c.properties['c:id']) - console.log('matched collections', collections) - console.log('queries before', JSON.stringify(this.queries)) let qs if (collections.length === 0) { qs = { bool: { must_not: { exists: { field: 'c:id' } } } } @@ -62,7 +58,7 @@ Search.prototype.search_items = (callback) => { } -Search.prototype.search_collections = (callback) => { +Search.prototype.search_collections = function (callback) { // hacky way to get all collections const sz = this.size const frm = this.frm @@ -92,7 +88,7 @@ Search.prototype.search_collections = (callback) => { } -Search.prototype.search = (index, callback) => { +Search.prototype.search = function (index, callback) { const self = this const searchParams = { @@ -129,20 +125,15 @@ Search.prototype.search = (index, callback) => { props = _.omit(props, ['bbox', 'geometry', 'assets', 'links', 'eo:bands']) const links = body.hits.hits[i]._source.links || [] // add self and collection links - const host = ( - 'X-Forwarded-Host' in self.headers ? - self.headers['X-Forwarded-Host'] : self.headers.Host - ) - const apiUrl = `${self.headers['X-Forwarded-Proto']}://${host}` let prefix = '/search/stac' if (index === 'collections') { prefix = '/collections' - links.self = { rel: 'self', href: `${apiUrl}${prefix}?c:id=${props.collection}` } + links.self = { rel: 'self', href: `${self.endpoint}${prefix}?c:id=${props.collection}` } } else { - links.self = { rel: 'self', href: `${apiUrl}${prefix}?id=${props.id}` } + links.self = { rel: 'self', href: `${self.endpoint}${prefix}?id=${props.id}` } if (_.has(props, 'c:id')) { - links.collection = { href: `${apiUrl}/collections/${props['c:id']}/definition` } + links.collection = { href: `${self.endpoint}/collections/${props['c:id']}/definition` } } } return { diff --git a/packages/api-lib/libs/es.js b/packages/api-lib/libs/es.js index 753f92e..763d36a 100644 --- a/packages/api-lib/libs/es.js +++ b/packages/api-lib/libs/es.js @@ -134,7 +134,7 @@ function streamToEs(stream, transform, client, index) { let nRecords = 0 let nTransformed = 0 - const toEs = through2({ objectMode: true, consume: true }, (data, encoding, next) => { + const toEs = through2({ objectMode: true, consume: true }, function (data, encoding, next) { const record = { index, type: 'doc', diff --git a/packages/api-lib/libs/ingest-csv.js b/packages/api-lib/libs/ingest-csv.js index fd60eea..d5d45a2 100644 --- a/packages/api-lib/libs/ingest-csv.js +++ b/packages/api-lib/libs/ingest-csv.js @@ -3,8 +3,6 @@ const csv = require('fast-csv') const AWS = require('aws-sdk') const zlib = require('zlib') -const pLimit = require('p-limit') -const lodash = require('lodash') const es = require('./es') let esClient @@ -34,10 +32,9 @@ function invokeLambda(bucket, key, nextFileNum, lastFileNum, arn, retries) { else { console.log(`launched ${JSON.stringify(params)}`) } - }).promise() + }) } - - return Promise.resolve() + return 0 } // split a CSV to multiple files and trigger lambdas @@ -128,7 +125,6 @@ function split({ newStream.on('error', (e) => cb(e)) return newStream.on('end', () => { - const limit = pLimit(3) //set concurrent call to 3 at a time // write the last records if (lineCounter > 0) { fileCounter += 1 @@ -158,16 +154,13 @@ function split({ `${extra} extra (Files ${startFile}-${maxEndFile})` ) - const promises = lodash.range(numLambdas).map((i) => limit(() => { - endFile = (i < extra) ? startFile + batchSize : (startFile + batchSize) - 1 + for (let i = 0; i < numLambdas; i += 1) { + endFile = (i < extra) ? (startFile + batchSize) : ((startFile + batchSize) - 1) + invokeLambda(bucket, key, startFile, Math.min(endFile, maxEndFile), arn) startFile = endFile + 1 - return invokeLambda(bucket, key, startFile - 1, Math.min(endFile, maxEndFile), arn) - })) - - return Promise.all(promises).then(() => cb()).catch(cb) + } } cb() - return Promise.resolve() }) } diff --git a/packages/api-lib/libs/queries.js b/packages/api-lib/libs/queries.js index dcda21f..505b0aa 100644 --- a/packages/api-lib/libs/queries.js +++ b/packages/api-lib/libs/queries.js @@ -87,7 +87,7 @@ const geometryQuery = (field, geometry) => { const intersects = (inGeojson, queries) => { - let geojson + let geojson = inGeojson // if we receive an object, assume it's GeoJSON, if not, try and parse if (typeof geojson === 'string') { try { diff --git a/packages/api-lib/package.json b/packages/api-lib/package.json index 33bb86c..9e9102d 100644 --- a/packages/api-lib/package.json +++ b/packages/api-lib/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/api-lib", - "version": "0.0.2", + "version": "0.1.0", "description": "A library for creating a search API of public Satellites metadata using Elasticsearch", "main": "index.js", "scripts": { diff --git a/packages/api/index.js b/packages/api/index.js index e8b4c33..6176d1f 100644 --- a/packages/api/index.js +++ b/packages/api/index.js @@ -11,7 +11,18 @@ module.exports.handler = (event, context, cb) => { // get payload const method = event.httpMethod - const payload = { query: {}, headers: event.headers } + let endpoint + if ('X-Forwarded-Host' in event.headers) { + endpoint = `${event.headers['X-Forwarded-Proto']}://${event.headers['X-Forwarded-Host']}` + } + else { + endpoint = `${event.headers['X-Forwarded-Proto']}://${event.headers.Host}` + if ('stage' in event.requestContext) { + endpoint = `${endpoint}/${event.requestContext.stage}` + } + } + + const payload = { query: {}, headers: event.headers, endpoint: endpoint } if (method === 'POST' && event.body) { payload.query = JSON.parse(event.body) } diff --git a/packages/api/package.json b/packages/api/package.json index d808faf..68c20bc 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/api", - "version": "0.0.2", + "version": "0.1.0", "description": "The api lambda function for sat-api", "main": "index.js", "repository": { @@ -20,11 +20,11 @@ }, "homepage": "https://github.com/sat-utils/sat-api#readme", "dependencies": { - "@sat-utils/api-lib": "^0.0.2", - "@sat-utils/ingest": "^0.0.2", - "@sat-utils/landsat": "^0.0.2", - "@sat-utils/manager": "^0.0.2", - "@sat-utils/sentinel": "^0.0.2", + "@sat-utils/api-lib": "^0.1.0", + "@sat-utils/ingest": "^0.1.0", + "@sat-utils/landsat": "^0.1.0", + "@sat-utils/manager": "^0.1.0", + "@sat-utils/sentinel": "^0.1.0", "lambda-proxy-utils": "^1.2.4", "lodash.get": "^4.4.2" }, diff --git a/packages/api/template/config.yml b/packages/api/template/config.yml index d41a3e3..a5cb428 100644 --- a/packages/api/template/config.yml +++ b/packages/api/template/config.yml @@ -38,7 +38,7 @@ default: source: 'node_modules/@sat-utils/ingest/dist/' handler: index.handler timeout: 300 - memory: 1024 + memory: 1600 envs: bucket: '{{system_bucket}}' prefix: '{{stackName}}' diff --git a/packages/ingest/index.js b/packages/ingest/index.js index 63c6b8f..cc68aa6 100644 --- a/packages/ingest/index.js +++ b/packages/ingest/index.js @@ -23,10 +23,11 @@ module.exports.handler = function handler(event, context, cb) { const arn = _.get(event, 'arn', '') let url - const reverse = true + let reverse = true switch (sat) { case 'landsat': url = 'https://landsat.usgs.gov/landsat/metadata_service/bulk_metadata_files/LANDSAT_8_C1.csv' + reverse = false break case 'sentinel': url = 'https://storage.googleapis.com/gcp-public-data-sentinel-2/index.csv.gz' @@ -40,7 +41,7 @@ module.exports.handler = function handler(event, context, cb) { bucket, key, arn, - maxFiles, + inMaxFiles: maxFiles, linesPerFile, maxLambdas, reverse, diff --git a/packages/ingest/package.json b/packages/ingest/package.json index ae4c248..30a7f4f 100644 --- a/packages/ingest/package.json +++ b/packages/ingest/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/ingest", - "version": "0.0.2", + "version": "0.1.0", "description": "ingest lambda function of sat-api", "main": "index.js", "repository": { @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/sat-utils/sat-api#readme", "dependencies": { - "@sat-utils/api-lib": "^0.0.2" + "@sat-utils/api-lib": "^0.1.0" }, "devDependencies": { "webpack": "~4.5.0", diff --git a/packages/landsat/index.js b/packages/landsat/index.js index f8d9319..cc43175 100644 --- a/packages/landsat/index.js +++ b/packages/landsat/index.js @@ -98,7 +98,7 @@ const collection = { function fileExists(url) { const params = { Bucket: 'landsat-pds', - Key: url.slice(36) + Key: url.slice(37) } return new Promise(((resolve, reject) => { s3.headObject(params, (err) => { @@ -136,8 +136,8 @@ function arrayIterate(values, fn) { function awsLinks(data) { // generates links for the data on AWS - const row = _.pad(data.row, 3, '0') - const _path = _.pad(data.path, 3, '0') + const row = _.padStart(data.row, 3, '0') + const _path = _.padStart(data.path, 3, '0') const sceneId = data.sceneID const productId = data.LANDSAT_PRODUCT_ID @@ -171,7 +171,8 @@ function awsLinks(data) { // check that file exists fileExists(c1.index).then(() => resolve(c1)) .catch((e) => { - const error = new Error(`${c1.index} not available: `) + console.log(`not avail: ${JSON.stringify(c1)}`) + const error = new Error(`${c1.index} not available: ${JSON.stringify(data)}`) reject(error, e) }) } @@ -180,7 +181,7 @@ function awsLinks(data) { let sid let newKey const rev = sceneId.slice(-2) - let prefix = `http://landsat-pds.s3.amazonaws.com/L8/${path.join(_path, row, _sceneId)}` + let prefix = `https://landsat-pds.s3.amazonaws.com/L8/${path.join(_path, row, _sceneId)}` const links = _.range(rev, -1, -1).map((r) => `${prefix}${_.pad(r, 2, '0')}/index.html`) arrayIterate(links.reverse(), fileExists).then((value) => { @@ -318,9 +319,13 @@ function handler(event, context, cb) { esClient = client return satlib.es.putMapping(esClient, 'collections') }) - .then(() => satlib.es.saveRecords(esClient, [collection], 'collections', 'c:id')) + .then(() => { + satlib.es.saveRecords(esClient, [collection], 'collections', 'c:id', (err) => { + if (err) console.log('Warning: ', err) + }) + }) .then(() => satlib.ingestcsv.update({ - esClient, + client: esClient, bucket, key, transform: _transform, @@ -330,7 +335,7 @@ function handler(event, context, cb) { arn, retries })) - .catch((e) => console.log('Error: ', e)) + .catch((e) => console.log(e)) } diff --git a/packages/landsat/package.json b/packages/landsat/package.json index ae8cade..0fb89bc 100644 --- a/packages/landsat/package.json +++ b/packages/landsat/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/landsat", - "version": "0.0.2", + "version": "0.1.0", "description": "Lambda function for ingesting landsat metadata to sat-api", "main": "index.js", "repository": { @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/sat-utils/sat-api#readme", "dependencies": { - "@sat-utils/api-lib": "^0.0.2", + "@sat-utils/api-lib": "^0.1.0", "got": "^8.3.2", "lodash": "^4.17.10", "moment": "^2.22.2", diff --git a/packages/manager/package.json b/packages/manager/package.json index ff97e01..259d553 100644 --- a/packages/manager/package.json +++ b/packages/manager/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/manager", - "version": "0.0.2", + "version": "0.1.0", "description": "Lambda function for managing elasticsearch of sat-api", "main": "index.js", "repository": { @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/sat-utils/sat-api#readme", "dependencies": { - "@sat-utils/api-lib": "^0.0.2" + "@sat-utils/api-lib": "^0.1.0" }, "devDependencies": { "webpack": "~4.5.0", diff --git a/packages/sentinel/index.js b/packages/sentinel/index.js index d793548..0e66388 100644 --- a/packages/sentinel/index.js +++ b/packages/sentinel/index.js @@ -140,19 +140,6 @@ function getTileUrl(tilePath) { } -function getSentinelInfo(url) { - /*return new Promise(function(resolve, reject) { - got(url, { json: true }).then(response => { - resolve(response) - }).catch(e => { - console.log(`error getting metadata: ${e}`) - resolve() - }) - })*/ - return got(url, { json: true }) -} - - function reproject(inputGeojson) { let geojson = clone(inputGeojson) const crs = geojson.crs.properties.name.replace( @@ -191,22 +178,23 @@ function transform(data, encoding, next) { const parsedMgrs = parseMgrs(mgrs) const tilePath = getTilePath(dt, parsedMgrs) const tileBaseUrl = getTileUrl(tilePath) + const rodaBaseUrl = tileBaseUrl.replace('sentinel-s2-l1c.s3.amazonaws.com', 'roda.sentinel-hub.com/sentinel-s2-l1c') const bands = range(1, 13).map((i) => pad(i, 3, 'B0')) bands.push('B8A') - getSentinelInfo(`${tileBaseUrl}/tileInfo.json`).then((body) => { - const info = body - const sat = info.productName.slice(0, 3) + const tileInfo = `${rodaBaseUrl}/tileInfo.json` + got(tileInfo, { json: true }).then((info) => { + const sat = info.body.productName.slice(0, 3) const satname = `Sentinel-2${sat.slice(-1)}` let val const files = fromPairs(bands.map((b) => { val = { href: `${tileBaseUrl}/${b}.jp2`, 'eo:bands': [b] } return [b, val] })) - files.thumbnail = { href: `${tileBaseUrl}/preview.jpg` } + files.thumbnail = { href: `${rodaBaseUrl}/preview.jpg` } files.tki = { href: `${tileBaseUrl}/TKI.jp2`, description: 'True Color Image' } - files.metadata = { href: `${tileBaseUrl}/metadata.xml` } + files.metadata = { href: `${rodaBaseUrl}/metadata.xml` } // reproject to EPSG:4326 - const geom = reproject(info.tileDataGeometry) + const geom = reproject(info.body.tileDataGeometry) const lons = geom.coordinates[0].map((pt) => pt[0]) const lats = geom.coordinates[0].map((pt) => pt[1]) const record = { @@ -251,9 +239,13 @@ function handler(event, context, cb) { esClient = client return satlib.es.putMapping(esClient, 'collections') }) - .then(() => satlib.es.saveRecords(esClient, [collection], 'collections', 'c:id')) + .then(() => { + satlib.es.saveRecords(esClient, [collection], 'collections', 'c:id', (err) => { + if (err) console.log('Warning: ', err) + }) + }) .then(() => satlib.ingestcsv.update({ - esClient, + client: esClient, bucket, key, transform: _transform, diff --git a/packages/sentinel/package.json b/packages/sentinel/package.json index 5fd0a0e..9656df1 100644 --- a/packages/sentinel/package.json +++ b/packages/sentinel/package.json @@ -1,6 +1,6 @@ { "name": "@sat-utils/sentinel", - "version": "0.0.2", + "version": "0.1.0", "description": "Lambda function for ingesting sentinel metadata to sat-api", "main": "index.js", "repository": { @@ -20,7 +20,7 @@ }, "homepage": "https://github.com/sat-utils/sat-api#readme", "dependencies": { - "@sat-utils/api-lib": "^0.0.2", + "@sat-utils/api-lib": "^0.1.0", "epsg": "^0.3.0", "got": "^8.3.2", "lodash.clonedeep": "^4.5.0",