Error during download (deployment zip)

Hi @hrishikesh,

Unfortunately, I need to download the whole site, but your link is useful nonetheless (If I end up having to download 2500 files manually :face_with_head_bandage:)

Please find the full script below:

const NetlifyAPI = require('netlify')
const Downloader = require('nodejs-file-downloader')
const client = new NetlifyAPI('');

(async () => {

  const files = await client.listSiteFiles({
    site_id: '54c85589-7d9a-4a30-b228-83d16cfa3165'
  })

  files.forEach(async (file) => {

    const downloader = new Downloader({
      url: 'superiorconcept.work' + file.path,
      directory: './download' + file.path.substring(0, file.path.lastIndexOf('/') + 1)
    })

    try {
      await downloader.download()
      console.log(file.path + ' done')
    } catch (error) {
      console.log(file.path + ' failed', error)
    }
  })

})()

I’ve made an additional mistake now, thinking I’d just be able to overwrite the current main index.html file with some additional CSS, but that’s not how it works (obviously…), and I ended up having a deploy with only one file; now I can see the script only tries to download the single file from the most recent deployment instead of the one that is published - however, the protocol mismatch issue still persists.

Any further information is greatly appreciated, thank you so much for looking into this!!

Ahh I suspected, you need to add the URL like https://superiorconcept.work

Additionally, I’ve a more complex version of this script drafted somewhere, but I didn’t release it publicly due to various issue related to its stability. Let me know if you want to give it a try.

Hi @hrishikesh,

The download came through now!

However, I’m still having an issue where the script only downloads the most recent deployment instead of the published one. The dashboard doesn’t seem to have the option to delete a manual deploy - would you be able to do this from your end?

Thanks a ton!

Hi @realCyberJunkie,

Deploys on Netlify can’t be deleted, unless you delete the site of course.

You can try this code:

const FS = require('fs')
const Chalk = require('chalk')
const Fetch = require('node-fetch')
const Inquirer = require('inquirer')
const NetlifyAPI = require('netlify')
const ByteSize = require('byte-size')
const Pluralize = require('pluralize')

const authToken = 'token'
const netlifyAuth = new NetlifyAPI(authToken)
const byteSizeOptions = {
  units: 'iec',
  precision: 2
}

netlifyAuth.listSites().then(sites => {

  console.log(Chalk.yellow('Fetching all websites...'))

  const siteList = []

  sites.forEach(site => {

    siteList.push({
      value: site.id,
      name: site.name
    })

  })

  Inquirer.prompt([{
    loop: false,
    name: 'site',
    type: 'list',
    pageSize: 10,
    choices: siteList,
    message: 'Choose a website:'
  }]).then(answer => {

    const siteName = siteList.filter(site => {
      return site.value === answer.site
    })

    netlifyAuth.listSiteDeploys({
      site_id: answer.site
    }).then(deploys => {

      console.log(Chalk.yellow('Fetching all successful deploys for the selected website...'))

      const deploysList = []

      deploys.forEach(deploy => {

        if (deploy.state != 'error' && !deploy.skipped) {

          deploysList.push({
            value: deploy.id,
            name: deploy.title
          })

        }

      })

      Inquirer.prompt([{
        loop: false,
        type: 'list',
        pageSize: 10,
        name: 'deploy',
        choices: deploysList,
        message: 'Choose a deploy:'
      }]).then(answer => {

        console.log(Chalk.yellow('Fetching all files for the selected deploy...'))

        const deployID = answer.deploy

        Fetch(`https://api.netlify.com/api/v1/deploys/${deployID}/files`, {
          headers: {
            Authorization: `Bearer ${authToken}`
          }
        }).then(response => response.json()).then(files => {

          let deploySize = 0
          const downloadableFiles = []

          let baseMarkup = `<!doctype html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1.0"><title>Deploy details for #${deployID}</title><style>*{font-family:sans-serif}table{border:1px solid black}table *{padding:5px}td:first-child,td:last-child{text-align:right}th:nth-child(2),td:nth-child(2){border:1px solid black;border-top:0;border-bottom:0}</style></head><body><p>View the <a href="https://app.netlify.com/sites/${siteName[0].name}/deploys/${deployID}/" target="_blank" rel="nofollow noopener noreferrer">deploy log</a>.</p><table><thead><tr><th>#</th><th>Path</th><th>Size</th></tr></thead><tbody>`

          files.forEach(file => {

            if (file.path != ('/netlify.toml' || '/_redirects' || '/_headers')) {
              downloadableFiles.push(file)
            }

          })

          downloadableFiles.forEach((file, index) => {
            deploySize += file.size
            baseMarkup += `<tr><td>${index + 1}</td><td><a href="https://${deployID}--${siteName[0].name}.netlify.app${file.path}" target="_blank" rel="nofollow noopener noreferrer">${file.path}</a></td><td>${ByteSize(file.size, byteSizeOptions)}</td></tr>`
          })

          baseMarkup += `</tbody></table><p>The deploy contains ${Pluralize('files', downloadableFiles.length, true)} amounting to ~${ByteSize(deploySize, byteSizeOptions)}</p></body></html>`

          FS.writeFileSync('./index.html', baseMarkup)

          console.log(Chalk.green('File list saved as ./index.html.'))

          Inquirer.prompt([{
            type: 'confirm',
            name: 'confirm',
            message: `Are you sure you wish to download the deploy? ${Chalk.red(`It will cost you ~${ByteSize(deploySize, byteSizeOptions)} of your Netlify bandwidth!`)}`
          }]).then(answer => {

            if (answer.confirm) {

              const downloadList = FS.createWriteStream('./files.txt')

              downloadableFiles.forEach(file => {
                downloadList.write(`https://${deployID}--${siteName[0].name}.netlify.app${file.path}\n\tdir=./download${file.path.substring(0, file.path.lastIndexOf('/') + 1).slice(0, -1)}\n`)
              })

              downloadList.end()

            } else {
              console.log(Chalk.green('Okay. Exiting...'))
            }

          })

        })

      })

    })

  })

})

You’d have to use aria2c to download the files: https://aria2.github.io. The files.txt file that gets generated is the one that you can ask aria2c to download. I had tried to integrate that tool in this code, but that never worked (it worked on Windows, not on macOS or Linux). So, I dropped the plan, but I believe this would still be useful for you.

You need to run npm i byte-size chalk inquirer netlify node-fetch pluralize to install all the required dependencies.

In this, you only need to add your API key, rest would be done by the code itself.

hi @hrishikesh,

I’m getting the following error with the script you provided:

const Fetch = require('node-fetch')
              ^

Error [ERR_REQUIRE_ESM]: require() of ES Module C:\Users\putin\Desktop\netlifydl\node_modules\node-fetch\src\index.js from C:\Users\putin\Desktop\netlifydl\index.js not supported.
Instead change the require of C:\Users\putin\Desktop\netlifydl\node_modules\node-fetch\src\index.js in C:\Users\putin\Desktop\netlifydl\index.js to a dynamic import() which is available in all CommonJS modules.
    at Object.<anonymous> (C:\Users\putin\Desktop\netlifydl\index.js:3:15) {
  code: 'ERR_REQUIRE_ESM'
}

When I tried changing const Fetch = require(‘node-fetch’) to const Fetch = import(‘node-fetch’), the script worked up to a point and printed the following error:

? Choose a deploy: 60700721258e817a0348c8ef
Fetching all files for the selected deploy...
C:\Users\putin\Desktop\netlifydl\index.js:78
        Fetch(`https://api.netlify.com/api/v1/deploys/${deployID}/files`, {
        ^

TypeError: Fetch is not a function
    at C:\Users\putin\Desktop\netlifydl\index.js:78:9
    at processTicksAndRejections (node:internal/process/task_queues:96:5)

Could you please help me figure this out?

Thank you!

Oh yeah, I believe node-fetch updated since the time I wrote this, could you try to use any 2.x.x release of that library?

1 Like

Hi @hrishikesh,

God Bless, I managed to download the whole site :grin: :grinning_face_with_smiling_eyes:

THANK YOU SO MUCH!!!

1 Like

Hi I’m trying to use your script, but I keep getting the following error:

const netlifyAuth = new NetlifyAPI(authToken)
                    ^

TypeError: NetlifyAPI is not a constructor

Any ideas why?

Hey there, @almostravioli :wave:

Thanks for chiming in here. Sorry you are encountering difficulties. Which site is this regarding?

Hello everyone,

I am experiencing the same issue. Error during download of recent deploy when I never had this issue downloading this same landing page several times. Zip file is not that large. I was building it on my netlify account and need to now download and upload the files into clients netlify and am at a loss :frowning: how to get the files.

Site ID is b9d0ed13-f793-4506-af09-fc56dfabf864

any help would be lovely. Thank you! - Ryan

Hey @ryan.mccarthy,

Could you try using the tool here:

https://netlify-file-browser.netlify.app/

Thank you so much @hrishikesh This link saved my life.
My cousin smashed my USB on the ground, and I lost the code for my whole Website, as well as its content.
You just saved me.
May god bless you:
https://netlify-file-browser.netlify.app/

Hi @hrishikesh

When files are too big for this tool - is there any other way to get them downloaded?

My website is very huge and this tool stops loading before it can even show the files. Are there any other ways to get these files downloaded?

You can use the Netlify API to create a CLI based tool. I have been working on it, but did not dedicate too much time to it so far. At the moment, I have this code:

import axios, {type AxiosError} from 'axios'
import {createWriteStream, existsSync, mkdirSync} from 'node:fs'
import {cwd} from 'node:process'
import {dirname, join} from 'node:path'
interface NFile {
  deploy_id : string
  id : string
  mime_type : string
  path : string
  sha : string
  site_id : string
  size : string
}
const axiosAbortController = new AbortController()
const axiosNetlify = axios.create({
  baseURL: 'https://api.netlify.com/api/v1',
  headers: {
    authorization: `Bearer token`
  },
  signal: axiosAbortController.signal
})
const deployId = 'id'
const fileList : Array<NFile> = []
function fetchFile(path : NFile['path']) {
  return axiosNetlify<ArrayBuffer>({
    headers: {
      'content-type': 'application/vnd.bitballoon.v1.raw'
    },
    responseType: 'arraybuffer',
    url: `https://api.netlify.com/api/v1/deploys/${deployId}/files/${path.slice(1)}`
  }).then(fileRes => {
    return fileRes.data
  }, (fileErr : AxiosError) => {

  })
}
function fetchFileList(page : number = 1) : Promise<void> {
  return axiosNetlify<Array<NFile>>({
    params: {
      page
    },
    url: `/deploys/${deployId}/files`
  }).then(fileListRes => {
    if (fileListRes.data.length === 100) {
      return fetchFileList(page + 1)
    } else {
      return
    }
  }, (fileListErr : AxiosError) => {

  })
}
fetchFileList().then(() => {
  const pathOnDisk = join(cwd(), './download/', fileList[0]!.path)
  fetchFile(fileList[0]!.path).then(data2 => {
    if (!existsSync(dirname(pathOnDisk))) {
      mkdirSync(dirname(pathOnDisk), {
        recursive: true
      })
    }
    createWriteStream(pathOnDisk).write(Buffer.from(data2))
  })
})

You can extend this code to make a downloader for yourself. If not, depending on how long I get to work on this, this tool would be released accordingly.

I’m assuming, CLI should be able to manage larger deploys as there would be no browser limitations + it would use less resources as there’s no GUI to render. So in theory the CLI should be able to manage deploys of any size, but can’t say until I complete this and try a deploy that’s fairly large.

1 Like

UPDATE: Here’s a tweaked version of that, which manages to download everything successfully:

import axios from 'axios'
import {createWriteStream, existsSync, mkdirSync} from 'node:fs'
import {cwd} from 'node:process'
import {dirname, join} from 'node:path'
interface NFile {
  deploy_id : string
  id : string
  mime_type : string
  path : string
  sha : string
  site_id : string
  size : number
}
const axiosAbortController = new AbortController()
const axiosNetlify = axios.create({
  baseURL: 'https://api.netlify.com/api/v1',
  headers: {
    authorization: `Bearer token`
  },
  signal: axiosAbortController.signal
})
const deployId = 'id'
let fileList : Array<NFile> = []
async function fetchFile(path : NFile['path']) : Promise<ArrayBuffer> {
  try {
    const fileRes = await axiosNetlify<ArrayBuffer>({
      headers: {
        'content-type': 'application/vnd.bitballoon.v1.raw'
      },
      responseType: 'arraybuffer',
      url: `https://api.netlify.com/api/v1/deploys/${deployId}/files/${path.slice(1)}`
    })
    return fileRes.data
  } catch {
    throw new Error(`failed to fetch ${path}`)
  }
}
async function fetchFileList(page : number = 1) {
  try {
    const fileListRes = await axiosNetlify<Array<NFile>>({
      params: {
        page
      },
      url: `/deploys/${deployId}/files`
    })
    fileList = fileList.concat(fileListRes.data)
    if (fileListRes.data.length === 100) {
      await fetchFileList(page + 1)
    }
  } catch {
    throw new Error('failed to fetch file list')
  }
}
await fetchFileList()
for (const file of fileList) {
  const fileBuffer = await fetchFile(file.path)
  const pathOnDisk = join(cwd(), './download/', file.path)
  const pathOnDiskDir = dirname(pathOnDisk)
  if (!existsSync(pathOnDiskDir)) {
    mkdirSync(pathOnDiskDir, {
      recursive: true
    })
  }
  createWriteStream(pathOnDisk).write(Buffer.from(fileBuffer))
}

It’s far from complete as it doesn’t show any useful info on if it’s working or not, but it will at least (probably) get you unblocked right now.

Thanks for that quick reply, do you only have this script in typescript? If so, how should the config file look like? at the moment I am facing this challenge:

You can remove the TS parts:

import axios from 'axios'
import {createWriteStream, existsSync, mkdirSync} from 'node:fs'
import {cwd} from 'node:process'
import {dirname, join} from 'node:path'
const axiosAbortController = new AbortController()
const axiosNetlify = axios.create({
  baseURL: 'https://api.netlify.com/api/v1',
  headers: {
    authorization: `Bearer token`
  },
  signal: axiosAbortController.signal
})
const deployId = 'id'
let fileList = []
async function fetchFile(path) {
  try {
    const fileRes = await axiosNetlify({
      headers: {
        'content-type': 'application/vnd.bitballoon.v1.raw'
      },
      responseType: 'arraybuffer',
      url: `https://api.netlify.com/api/v1/deploys/${deployId}/files/${path.slice(1)}`
    })
    return fileRes.data
  } catch {
    throw new Error(`failed to fetch ${path}`)
  }
}
async function fetchFileList(page = 1) {
  try {
    const fileListRes = await axiosNetlify({
      params: {
        page
      },
      url: `/deploys/${deployId}/files`
    })
    fileList = fileList.concat(fileListRes.data)
    if (fileListRes.data.length === 100) {
      await fetchFileList(page + 1)
    }
  } catch {
    throw new Error('failed to fetch file list')
  }
}
await fetchFileList()
for (const file of fileList) {
  const fileBuffer = await fetchFile(file.path)
  const pathOnDisk = join(cwd(), './download/', file.path)
  const pathOnDiskDir = dirname(pathOnDisk)
  if (!existsSync(pathOnDiskDir)) {
    mkdirSync(pathOnDiskDir, {
      recursive: true
    })
  }
  createWriteStream(pathOnDisk).write(Buffer.from(fileBuffer))
}

Note that, since it downloads one file at a time, it’s going to take really long to download the entire deploy. My deploy of about 1200 files took about or more than an hour. You can customise the download logic in the last lines of the script, but then you might need to be cautious about rate limits on the API.

Ok, Thanks. without TS it seems to work except that now my only blocker is the deployID. Initially I thought it’s the site ID but that throws the (‘failed to fetch file list’) error. My site was renamed, so I do not have that default url to see the deplyID. It was a manual upload and at this point I am not sure where else can I get the deployID?

The deploy ID is shown in the URL: https://app.netlify.com/sites/<site-name>/deploys/<deploy-id> whenever you go to a deploy page.

Thanks, I got the ID but I am still not getting the files. That (‘failed to fetch file list’) error still shows up and I am sure that I am using the right deploy ID and authorization value. Access token is the authorization value, right? I wonder what could I be doing wrong… how did you see the number of files in your deploy? my deploy logs have no messages