Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Caching#76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Open
jonsharratt wants to merge3 commits intomaster
base:master
Choose a base branch
Loading
fromcache
Open
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 34 additions & 7 deletions.serverless_plugins/remove-storage/index.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -28,23 +28,29 @@ class RemoveStorageBucket {
.Properties
.BucketName;

this.bucketCache = this.serverless.service.resources
.Resources
.PackageCacheStorage
.Properties
.BucketName;

this.hooks = {
'before:remove:remove': this.beforeRemove.bind(this),
};
}

listAllKeys(token) {
listAllKeys(bucket,token) {
const allKeys = [];
return this.s3.listObjectsV2({
Bucket:this.bucket,
Bucket: bucket,
ContinuationToken: token,
})
.promise()
.then((data) => {
allKeys.push(data.Contents);

if (data.IsTruncated) {
return this.listAllKeys(data.NextContinuationToken);
return this.listAllKeys(bucket,data.NextContinuationToken);
}

return [].concat(...allKeys).map(({ Key }) => ({ Key }));
Expand All@@ -53,19 +59,40 @@ class RemoveStorageBucket {

beforeRemove() {
return new Promise((resolve, reject) => {
returnthis.listAllKeys()
this.listAllKeys(this.bucketCache)
.then((keys) => {
if (keys.length > 0) {
return this.s3
.deleteObjects({
Bucket: this.bucket,
Bucket: this.bucketCache,
Delete: {
Objects: keys,
},
}).promise();
}

return true;
})
.then(() => {
return this.s3
.deleteBucket({
Bucket: this.bucketCache,
}).promise()
.then(() => {
this.serverless.cli.log('AWS Package Storage Cache Removed');
});
})
.then(() => {
return this.listAllKeys(this.bucket)
.then((keys) => {
if (keys.length > 0) {
return this.s3
.deleteObjects({
Bucket: this.bucket,
Delete: {
Objects: keys,
},
}).promise();
}
})
})
.then(() => {
return this.s3
Expand Down
78 changes: 78 additions & 0 deletions.serverless_plugins/update-cache-env/index.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
const util = require('util');

class UpdateCacheEnv {
constructor(serverless, options) {
this.options = options;
this.serverless = serverless;
this.provider = this.serverless.getProvider('aws');

this.awsInfo = this.serverless
.pluginManager
.plugins
.find(p => p.constructor.name === 'AwsInfo');

this.bucketCache = this.serverless
.service
.provider
.environment
.bucketCache;

this.registry = this.serverless
.service
.provider
.environment
.registry;

this.cacheEnabled = this.serverless
.service
.provider
.environment
.cacheEnabled;

this.hooks = {
'after:deploy:deploy': this.afterDeploy.bind(this),
};
}

afterDeploy() {
const lambda = new this.provider.sdk.Lambda({
signatureVersion: 'v4',
region: this.options.region,
});

const cacheFunction = this
.awsInfo
.gatheredData
.info
.functions
.find(f => f.name === 'cache');

if (!cacheFunction) {
throw new Error('Cache function has not been deployed correctly to AWS.');
}

const params = {
FunctionName: cacheFunction.deployedName,
Environment: {
Variables: {
apiEndpoint: `${this.awsInfo.gatheredData.info.endpoint}/registry`,
region: this.options.region,
bucketCache: this.bucketCache,
registry: this.registry,
cacheEnabled: this.cacheEnabled,
}
},
};

lambda
.updateFunctionConfiguration(params)
.promise()
.then(() => {
this.serverless.cli.log('AWS Cache Environment Ready.');
}).catch(err => {
this.serverless.cli.log(err.message);
});
}
}

module.exports = UpdateCacheEnv;
6 changes: 6 additions & 0 deletionsREADME.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,6 +32,7 @@ export CODEBOX_GITHUB_URL="https://api.github.com/" # The GitHub / GitHub Enterp
export CODEBOX_GITHUB_CLIENT_ID="client_id" # The client id for your GitHub application
export CODEBOX_GITHUB_SECRET="secret" # The secret for your GitHub application
export CODEBOX_RESTRICTED_ORGS="" # OPTIONAL: Comma seperated list of github organisations to only allow access to users in that org (e.g. "craftship,myorg"). Useful if using public GitHub for authentication, as by default all authenticated users would have access.
export CODEBOX_CACHE="false" # OPTIONAL: Any npm install will cache dependencies from public registry in a separate S3 bucket
```
* `serverless deploy --stage prod` (pick which ever stage you wish)
* `npm set registry <url>` - `<url>` being the base url shown in the terminal after deployment completes, such as:
Expand DownExpand Up@@ -64,6 +65,11 @@ Once done ensure you have a project based `.npmrc` config setup a per the "Using

Yarn does not require an explicit `yarn login` as in this scenario it uses your `.npmrc` config instead.

## Caching
If you enable `CODEBOX_CACHE="true"` when using the registry all requests to packages that hit the public registry will then be cached. This allows you to have a cache / mirror of all dependencies in your project. Helps with robust deployments and better response times when hosting your CI in the same region as your Codebox npm registry.

**NOTE: Your AWS bill will rise due to this scheduled task ensuring that cached dependnecies are up to date.**

## Admins / Publishing Packages
`npm publish` works as it normally does via the npm CLI. By default all users that authenticate have read only access. If you wish to allow publish rights then you need to set the `CODEBOX_ADMINS` environment variable to a comma separated list of GitHub usernames such as `jonsharratt,kadikraman` and re-deploy.

Expand Down
15 changes: 15 additions & 0 deletionsserverless.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ plugins:
- environment-variables
- remove-storage
- serverless-webpack
- update-cache-env
- content-handling
- codebox-tools

Expand All@@ -22,7 +23,9 @@ provider:
githubClientId: ${env:CODEBOX_GITHUB_CLIENT_ID}
githubSecret: ${env:CODEBOX_GITHUB_SECRET}
bucket: ${env:CODEBOX_BUCKET}-${self:provider.stage}
bucketCache: ${env:CODEBOX_BUCKET}-cache-${self:provider.stage}
region: ${self:provider.region}
cacheEnabled: ${env:CODEBOX_CACHE}

clientId: ${env:CODEBOX_INSIGHTS_CLIENT_ID}
secret: ${env:CODEBOX_INSIGHTS_SECRET}
Expand All@@ -38,6 +41,7 @@ provider:
- "sns:Publish"
Resource:
- "arn:aws:s3:::${self:provider.environment.bucket}*"
- "arn:aws:s3:::${self:provider.environment.bucketCache}*"
- "Fn::Join":
- ""
- - "arn:aws:sns:"
Expand All@@ -50,6 +54,12 @@ functions:
authorizerGithub:
handler: authorizerGithub.default

cache:
handler: cache.default
timeout: 300
events:
- schedule: rate(1 hour)

put:
handler: put.default
events:
Expand DownExpand Up@@ -134,6 +144,11 @@ resources:
Properties:
AccessControl: Private
BucketName: ${self:provider.environment.bucket}
PackageCacheStorage:
Type: AWS::S3::Bucket
Properties:
AccessControl: Private
BucketName: ${self:provider.environment.bucketCache}

custom:
webpackIncludeModules: true
16 changes: 16 additions & 0 deletionssrc/adapters/s3.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,4 +25,20 @@ export default class Storage {

return meta.Body;
}

async listAllKeys(token = null, keys = []) {
const data = await this.S3.listObjectsV2({
ContinuationToken: token,
})
.promise();

keys.push(data.Contents);

if (data.IsTruncated) {
return this.listAllKeys(data.NextContinuationToken, keys);
}

return [].concat(...keys).map(({ Key }) => Key);
}
}

4 changes: 4 additions & 0 deletionssrc/contextFactory.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,9 @@ const log = (cmd, namespace, region, topic) => {
export default (namespace, { headers, requestContext }) => {
const {
registry,
cacheEnabled,
bucket,
bucketCache,
region,
logTopic,
} = process.env;
Expand All@@ -62,9 +64,11 @@ export default (namespace, { headers, requestContext }) => {

return {
command: cmd,
cacheEnabled: (cacheEnabled === 'true'),
registry,
user: user(requestContext.authorizer),
storage: storage(region, bucket),
cache: storage(region, bucketCache),
log: log(cmd, namespace, region, logTopic),
npm,
};
Expand Down
Loading
[8]ページ先頭
©2009-2025 Movatter.jp