mirror of
https://github.com/felt/tippecanoe.git
synced 2026-10-02 08:25:40 +02:00
Remove unused Dockerfiles and lambda to avoid security warnings (#365)
This commit is contained in:
-110
@@ -1,110 +0,0 @@
|
|||||||
language: node_js
|
|
||||||
node_js:
|
|
||||||
- "6"
|
|
||||||
|
|
||||||
sudo: false
|
|
||||||
|
|
||||||
matrix:
|
|
||||||
include:
|
|
||||||
# test on docker+centos7
|
|
||||||
- os: linux
|
|
||||||
compiler: clang
|
|
||||||
services:
|
|
||||||
- docker
|
|
||||||
sudo: true
|
|
||||||
dist: trusty
|
|
||||||
env: DOCKERFILE=Dockerfile.centos7
|
|
||||||
before_install: []
|
|
||||||
install:
|
|
||||||
- docker build -t tippecanoe-image -f ${DOCKERFILE} .
|
|
||||||
script:
|
|
||||||
- docker run -it tippecanoe-image
|
|
||||||
# test on docker+ubuntu
|
|
||||||
- os: linux
|
|
||||||
compiler: clang
|
|
||||||
services:
|
|
||||||
- docker
|
|
||||||
sudo: true
|
|
||||||
dist: trusty
|
|
||||||
env: DOCKERFILE=Dockerfile
|
|
||||||
before_install: []
|
|
||||||
install:
|
|
||||||
- docker build -t tippecanoe-image -f ${DOCKERFILE} .
|
|
||||||
script:
|
|
||||||
- docker run -it tippecanoe-image
|
|
||||||
# debug+integer-santizer build
|
|
||||||
- os: linux
|
|
||||||
compiler: clang
|
|
||||||
env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=integer" CFLAGS="-fsanitize=integer" LDFLAGS="-fsanitize=integer"
|
|
||||||
addons:
|
|
||||||
apt:
|
|
||||||
sources: ['ubuntu-toolchain-r-test' ]
|
|
||||||
packages: [ 'libstdc++6','libstdc++-5-dev' ]
|
|
||||||
# debug+leak+address-sanitizer build
|
|
||||||
- os: linux
|
|
||||||
compiler: clang
|
|
||||||
env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug ASAN_OPTIONS=detect_leaks=1 CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="-fsanitize=address,undefined" CFLAGS="-fsanitize=address,undefined" LDFLAGS="-fsanitize=address,undefined" FEWER=true
|
|
||||||
addons:
|
|
||||||
apt:
|
|
||||||
sources: ['ubuntu-toolchain-r-test' ]
|
|
||||||
packages: [ 'libstdc++6','libstdc++-5-dev' ]
|
|
||||||
# coverage+debug build
|
|
||||||
- os: linux
|
|
||||||
compiler: clang
|
|
||||||
env: CLANG_VERSION='3.8.0' BUILDTYPE=Debug CC="clang-3.8" CXX="clang++-3.8" CXXFLAGS="--coverage" CFLAGS="--coverage" LDFLAGS="--coverage"
|
|
||||||
after_script:
|
|
||||||
- mason install llvm-cov 3.9.1
|
|
||||||
- mason link llvm-cov 3.9.1
|
|
||||||
- which llvm-cov
|
|
||||||
- curl -S -f https://codecov.io/bash -o codecov
|
|
||||||
- chmod +x codecov
|
|
||||||
- ./codecov -x "llvm-cov gcov" -Z
|
|
||||||
addons:
|
|
||||||
apt:
|
|
||||||
sources: ['ubuntu-toolchain-r-test' ]
|
|
||||||
packages: [ 'libstdc++6','libstdc++-5-dev' ]
|
|
||||||
# release+linux+g++
|
|
||||||
- os: linux
|
|
||||||
compiler: gcc
|
|
||||||
env: BUILDTYPE=Release CC="gcc-4.9" CXX="g++-4.9"
|
|
||||||
addons:
|
|
||||||
apt:
|
|
||||||
sources: ['ubuntu-toolchain-r-test']
|
|
||||||
packages: [ 'g++-4.9' ]
|
|
||||||
# release+linux+clang++
|
|
||||||
- os: linux
|
|
||||||
compiler: clang
|
|
||||||
env: CLANG_VERSION='3.8.0' BUILDTYPE=Release CC="clang-3.8" CXX="clang++-3.8"
|
|
||||||
addons:
|
|
||||||
apt:
|
|
||||||
sources: ['ubuntu-toolchain-r-test' ]
|
|
||||||
packages: [ 'libstdc++6','libstdc++-5-dev' ]
|
|
||||||
# release+osx
|
|
||||||
- os: osx
|
|
||||||
compiler: clang
|
|
||||||
env: BUILDTYPE=Release
|
|
||||||
# debug+osx
|
|
||||||
- os: osx
|
|
||||||
compiler: clang
|
|
||||||
env: BUILDTYPE=Debug
|
|
||||||
|
|
||||||
before_install:
|
|
||||||
- DEPS_DIR="${TRAVIS_BUILD_DIR}/deps"
|
|
||||||
- export PATH=${DEPS_DIR}/bin:${PATH} && mkdir -p ${DEPS_DIR}
|
|
||||||
- |
|
|
||||||
if [[ ${CLANG_VERSION:-false} != false ]]; then
|
|
||||||
export CCOMPILER='clang'
|
|
||||||
export CXXCOMPILER='clang++'
|
|
||||||
CLANG_URL="https://mason-binaries.s3.amazonaws.com/${TRAVIS_OS_NAME}-x86_64/clang/${CLANG_VERSION}.tar.gz"
|
|
||||||
travis_retry wget --quiet -O - ${CLANG_URL} | tar --strip-components=1 -xz -C ${DEPS_DIR}
|
|
||||||
fi
|
|
||||||
|
|
||||||
install:
|
|
||||||
- BUILDTYPE=${BUILDTYPE} make -j
|
|
||||||
|
|
||||||
script:
|
|
||||||
- npm install geobuf
|
|
||||||
- if [ -n "${FEWER}" ]; then
|
|
||||||
BUILDTYPE=${BUILDTYPE} make fewer-tests; else
|
|
||||||
BUILDTYPE=${BUILDTYPE} make test geobuf-test;
|
|
||||||
fi
|
|
||||||
-20
@@ -1,20 +0,0 @@
|
|||||||
FROM ubuntu:22.04 AS tippecanoe-builder
|
|
||||||
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get -y install make gcc g++ libsqlite3-dev zlib1g-dev
|
|
||||||
|
|
||||||
COPY . /tmp/tippecanoe-src
|
|
||||||
WORKDIR /tmp/tippecanoe-src
|
|
||||||
|
|
||||||
RUN make
|
|
||||||
|
|
||||||
CMD make test
|
|
||||||
|
|
||||||
# Using multistage build reduces the docker image size by alot by only copying the needed binaries
|
|
||||||
FROM ubuntu:22.04
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get -y install libsqlite3-0 \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
COPY --from=tippecanoe-builder /tmp/tippecanoe-src/tippecanoe* /usr/local/bin/
|
|
||||||
COPY --from=tippecanoe-builder /tmp/tippecanoe-src/tile-join /usr/local/bin/
|
|
||||||
WORKDIR /app
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
FROM centos:centos8
|
|
||||||
|
|
||||||
RUN yum install -y make sqlite-devel zlib-devel bash git gcc-c++
|
|
||||||
|
|
||||||
# Create a directory and copy in all files
|
|
||||||
RUN mkdir -p /tmp/tippecanoe-src
|
|
||||||
WORKDIR /tmp/tippecanoe-src
|
|
||||||
COPY . /tmp/tippecanoe-src
|
|
||||||
|
|
||||||
# Build tippecanoe
|
|
||||||
RUN make \
|
|
||||||
&& make install
|
|
||||||
|
|
||||||
# Run the tests
|
|
||||||
CMD make test
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
*.zip
|
|
||||||
tmp/*
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
# Build the tippecanoe layer (binary)
|
|
||||||
|
|
||||||
1. On an AWS Linux 2 instance or image:
|
|
||||||
|
|
||||||
sudo yum update
|
|
||||||
sudo yum install git clang sqlite-devel zlib-devel
|
|
||||||
|
|
||||||
2. Create a ZIP archive with the `tippecanoe` binary at `bin/tippecanoe` and upload as a Lambda layer
|
|
||||||
3. Copy [tippecanoe.py](tippecanoe.py) (python 3.9 runtime)
|
|
||||||
4. set the environment variable `OUTPUT_BUCKET` to your output bucket; give lambda IAM role access to input/output buckets
|
|
||||||
5. Enable ACLs on the output bucket
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"Version": "2012-10-17",
|
|
||||||
"Statement": [
|
|
||||||
{
|
|
||||||
"Sid": "AllowGet",
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Action": "s3:GetObject",
|
|
||||||
"Resource": "arn:aws:s3:::protomaps-tippecanoe-test/*"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"Sid": "AllowPut",
|
|
||||||
"Effect": "Allow",
|
|
||||||
"Action": [
|
|
||||||
"s3:PutObject",
|
|
||||||
"s3:PutObjectAcl"
|
|
||||||
],
|
|
||||||
"Resource": "arn:aws:s3:::protomaps-tippecanoe-test-output/*"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
boto3==1.20.32
|
|
||||||
botocore==1.23.32
|
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
import os
|
|
||||||
import json
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import boto3
|
|
||||||
from botocore import UNSIGNED
|
|
||||||
from botocore.config import Config
|
|
||||||
|
|
||||||
# if the input bucket is publicly readable
|
|
||||||
#s3 = boto3.client('s3',config=Config(signature_version=UNSIGNED))
|
|
||||||
|
|
||||||
def lambda_handler(event, context):
|
|
||||||
bucket_name = event['Records'][0]['s3']['bucket']['name']
|
|
||||||
input_key = event['Records'][0]['s3']['object']['key']
|
|
||||||
tmpdir = '/tmp'
|
|
||||||
run_tippecanoe('tippecanoe',tmpdir,bucket_name, input_key)
|
|
||||||
return {
|
|
||||||
'statusCode': 200,
|
|
||||||
'body': json.dumps('Done!')
|
|
||||||
}
|
|
||||||
|
|
||||||
# input formats: "geojsonseq", "fgb", "geobuf", "geojson", "csv"
|
|
||||||
def split_key(key):
|
|
||||||
if key.endswith(".gz"):
|
|
||||||
root, ext = os.path.splitext(key[0:-3])
|
|
||||||
return root, ext + ".gz"
|
|
||||||
else:
|
|
||||||
return os.path.splitext(key)
|
|
||||||
|
|
||||||
def run_tippecanoe(executable, tmpdir, bucket_name, input_key):
|
|
||||||
s3 = boto3.client('s3')
|
|
||||||
root, ext = split_key(input_key)
|
|
||||||
input_path = os.path.join(tmpdir,'input' + ext)
|
|
||||||
output_path = os.path.join(tmpdir,'output.mbtiles')
|
|
||||||
s3.download_file(bucket_name,input_key,input_path)
|
|
||||||
p = subprocess.Popen([executable,'-o',output_path,input_path,'--force','-u','-U','5'],stderr=subprocess.PIPE)
|
|
||||||
for line in p.stderr:
|
|
||||||
try:
|
|
||||||
j = json.loads(line)
|
|
||||||
s3.put_object(Body=json.dumps(j),Bucket=os.environ['OUTPUT_BUCKET'],Key=root + ".progress",ACL="public-read",ContentType="application/json",ContentDisposition="inline")
|
|
||||||
except json.decoder.JSONDecodeError:
|
|
||||||
pass
|
|
||||||
s3.upload_file(output_path,os.environ['OUTPUT_BUCKET'],root + ".mbtiles")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
tmpdir = 'tmp'
|
|
||||||
bucket_name = sys.argv[1]
|
|
||||||
input_key = sys.argv[2]
|
|
||||||
run_tippecanoe('../tippecanoe', tmpdir, bucket_name, input_key)
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
from tippecanoe import split_key
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
class TestTippecanoe(unittest.TestCase):
|
|
||||||
def test_basic(self):
|
|
||||||
root, ext = split_key("file.json")
|
|
||||||
self.assertEqual(root,"file")
|
|
||||||
self.assertEqual(ext,".json")
|
|
||||||
|
|
||||||
def test_basic_gz(self):
|
|
||||||
root, ext = split_key("file.csv.gz")
|
|
||||||
self.assertEqual(root,"file")
|
|
||||||
self.assertEqual(ext,".csv.gz")
|
|
||||||
|
|
||||||
def test_nested_key(self):
|
|
||||||
root, ext = split_key("dir/file.fgb")
|
|
||||||
self.assertEqual(root,"dir/file")
|
|
||||||
self.assertEqual(ext,".fgb")
|
|
||||||
|
|
||||||
def test_nested_key_gz(self):
|
|
||||||
root, ext = split_key("dir/file.geojsonseq.gz")
|
|
||||||
self.assertEqual(root,"dir/file")
|
|
||||||
self.assertEqual(ext,".geojsonseq.gz")
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
||||||
Reference in New Issue
Block a user