cache.js 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. 'use strict'
  2. const cacache = require('cacache')
  3. const fetch = require('node-fetch-npm')
  4. const pipe = require('mississippi').pipe
  5. const ssri = require('ssri')
  6. const through = require('mississippi').through
  7. const to = require('mississippi').to
  8. const url = require('url')
  9. const stream = require('stream')
  10. const MAX_MEM_SIZE = 5 * 1024 * 1024 // 5MB
  11. function cacheKey (req) {
  12. const parsed = url.parse(req.url)
  13. return `make-fetch-happen:request-cache:${
  14. url.format({
  15. protocol: parsed.protocol,
  16. slashes: parsed.slashes,
  17. host: parsed.host,
  18. hostname: parsed.hostname,
  19. pathname: parsed.pathname
  20. })
  21. }`
  22. }
  23. // This is a cacache-based implementation of the Cache standard,
  24. // using node-fetch.
  25. // docs: https://developer.mozilla.org/en-US/docs/Web/API/Cache
  26. //
  27. module.exports = class Cache {
  28. constructor (path, opts) {
  29. this._path = path
  30. this.Promise = (opts && opts.Promise) || Promise
  31. }
  32. // Returns a Promise that resolves to the response associated with the first
  33. // matching request in the Cache object.
  34. match (req, opts) {
  35. opts = opts || {}
  36. const key = cacheKey(req)
  37. return cacache.get.info(this._path, key).then(info => {
  38. return info && cacache.get.hasContent(
  39. this._path, info.integrity, opts
  40. ).then(exists => exists && info)
  41. }).then(info => {
  42. if (info && info.metadata && matchDetails(req, {
  43. url: info.metadata.url,
  44. reqHeaders: new fetch.Headers(info.metadata.reqHeaders),
  45. resHeaders: new fetch.Headers(info.metadata.resHeaders),
  46. cacheIntegrity: info.integrity,
  47. integrity: opts && opts.integrity
  48. })) {
  49. const resHeaders = new fetch.Headers(info.metadata.resHeaders)
  50. addCacheHeaders(resHeaders, this._path, key, info.integrity, info.time)
  51. if (req.method === 'HEAD') {
  52. return new fetch.Response(null, {
  53. url: req.url,
  54. headers: resHeaders,
  55. status: 200
  56. })
  57. }
  58. let body
  59. const cachePath = this._path
  60. // avoid opening cache file handles until a user actually tries to
  61. // read from it.
  62. if (opts.memoize !== false && info.size > MAX_MEM_SIZE) {
  63. body = new stream.PassThrough()
  64. const realRead = body._read
  65. body._read = function (size) {
  66. body._read = realRead
  67. pipe(
  68. cacache.get.stream.byDigest(cachePath, info.integrity, {
  69. memoize: opts.memoize
  70. }),
  71. body,
  72. err => body.emit(err))
  73. return realRead.call(this, size)
  74. }
  75. } else {
  76. let readOnce = false
  77. // cacache is much faster at bulk reads
  78. body = new stream.Readable({
  79. read () {
  80. if (readOnce) return this.push(null)
  81. readOnce = true
  82. cacache.get.byDigest(cachePath, info.integrity, {
  83. memoize: opts.memoize
  84. }).then(data => {
  85. this.push(data)
  86. this.push(null)
  87. }, err => this.emit('error', err))
  88. }
  89. })
  90. }
  91. return this.Promise.resolve(new fetch.Response(body, {
  92. url: req.url,
  93. headers: resHeaders,
  94. status: 200,
  95. size: info.size
  96. }))
  97. }
  98. })
  99. }
  100. // Takes both a request and its response and adds it to the given cache.
  101. put (req, response, opts) {
  102. opts = opts || {}
  103. const size = response.headers.get('content-length')
  104. const fitInMemory = !!size && opts.memoize !== false && size < MAX_MEM_SIZE
  105. const ckey = cacheKey(req)
  106. const cacheOpts = {
  107. algorithms: opts.algorithms,
  108. metadata: {
  109. url: req.url,
  110. reqHeaders: req.headers.raw(),
  111. resHeaders: response.headers.raw()
  112. },
  113. size,
  114. memoize: fitInMemory && opts.memoize
  115. }
  116. if (req.method === 'HEAD' || response.status === 304) {
  117. // Update metadata without writing
  118. return cacache.get.info(this._path, ckey).then(info => {
  119. // Providing these will bypass content write
  120. cacheOpts.integrity = info.integrity
  121. addCacheHeaders(
  122. response.headers, this._path, ckey, info.integrity, info.time
  123. )
  124. return new this.Promise((resolve, reject) => {
  125. pipe(
  126. cacache.get.stream.byDigest(this._path, info.integrity, cacheOpts),
  127. cacache.put.stream(this._path, cacheKey(req), cacheOpts),
  128. err => err ? reject(err) : resolve(response)
  129. )
  130. })
  131. }).then(() => response)
  132. }
  133. let buf = []
  134. let bufSize = 0
  135. let cacheTargetStream = false
  136. const cachePath = this._path
  137. let cacheStream = to((chunk, enc, cb) => {
  138. if (!cacheTargetStream) {
  139. if (fitInMemory) {
  140. cacheTargetStream =
  141. to({highWaterMark: MAX_MEM_SIZE}, (chunk, enc, cb) => {
  142. buf.push(chunk)
  143. bufSize += chunk.length
  144. cb()
  145. }, done => {
  146. cacache.put(
  147. cachePath,
  148. cacheKey(req),
  149. Buffer.concat(buf, bufSize),
  150. cacheOpts
  151. ).then(
  152. () => done(),
  153. done
  154. )
  155. })
  156. } else {
  157. cacheTargetStream =
  158. cacache.put.stream(cachePath, cacheKey(req), cacheOpts)
  159. }
  160. }
  161. cacheTargetStream.write(chunk, enc, cb)
  162. }, done => {
  163. cacheTargetStream ? cacheTargetStream.end(done) : done()
  164. })
  165. const oldBody = response.body
  166. const newBody = through({highWaterMark: MAX_MEM_SIZE})
  167. response.body = newBody
  168. oldBody.once('error', err => newBody.emit('error', err))
  169. newBody.once('error', err => oldBody.emit('error', err))
  170. cacheStream.once('error', err => newBody.emit('error', err))
  171. pipe(oldBody, to((chunk, enc, cb) => {
  172. cacheStream.write(chunk, enc, () => {
  173. newBody.write(chunk, enc, cb)
  174. })
  175. }, done => {
  176. cacheStream.end(() => {
  177. newBody.end(() => {
  178. done()
  179. })
  180. })
  181. }), err => err && newBody.emit('error', err))
  182. return response
  183. }
  184. // Finds the Cache entry whose key is the request, and if found, deletes the
  185. // Cache entry and returns a Promise that resolves to true. If no Cache entry
  186. // is found, it returns false.
  187. 'delete' (req, opts) {
  188. opts = opts || {}
  189. if (typeof opts.memoize === 'object') {
  190. if (opts.memoize.reset) {
  191. opts.memoize.reset()
  192. } else if (opts.memoize.clear) {
  193. opts.memoize.clear()
  194. } else {
  195. Object.keys(opts.memoize).forEach(k => {
  196. opts.memoize[k] = null
  197. })
  198. }
  199. }
  200. return cacache.rm.entry(
  201. this._path,
  202. cacheKey(req)
  203. // TODO - true/false
  204. ).then(() => false)
  205. }
  206. }
  207. function matchDetails (req, cached) {
  208. const reqUrl = url.parse(req.url)
  209. const cacheUrl = url.parse(cached.url)
  210. const vary = cached.resHeaders.get('Vary')
  211. // https://tools.ietf.org/html/rfc7234#section-4.1
  212. if (vary) {
  213. if (vary.match(/\*/)) {
  214. return false
  215. } else {
  216. const fieldsMatch = vary.split(/\s*,\s*/).every(field => {
  217. return cached.reqHeaders.get(field) === req.headers.get(field)
  218. })
  219. if (!fieldsMatch) {
  220. return false
  221. }
  222. }
  223. }
  224. if (cached.integrity) {
  225. return ssri.parse(cached.integrity).match(cached.cacheIntegrity)
  226. }
  227. reqUrl.hash = null
  228. cacheUrl.hash = null
  229. return url.format(reqUrl) === url.format(cacheUrl)
  230. }
  231. function addCacheHeaders (resHeaders, path, key, hash, time) {
  232. resHeaders.set('X-Local-Cache', encodeURIComponent(path))
  233. resHeaders.set('X-Local-Cache-Key', encodeURIComponent(key))
  234. resHeaders.set('X-Local-Cache-Hash', encodeURIComponent(hash))
  235. resHeaders.set('X-Local-Cache-Time', new Date(time).toUTCString())
  236. }