Site name: ae-prod (audioeye.com)
Recently started seeing an elusive issue related to split testing which I suspect could be Netlify related.
Context: We have 2 branches being served using split testing, each at 50% distribution. We’re using Gatsby V4.7.1, and I’ll post the package.json at the bottom of this post. I have successfully ran split tests without this issue for the past few months, but something has changed in the last 1-2 weeks that has created issues.
We started noticing within the last few days that we are often seeing 400’s and 404’s on individual Webpack bundled JS files, causing errors and partial page loading issues, but only one on of the two branches, and not always the same branch.
Unfortunately, this behavior has been hard to reproduce consistently, but I have confirmed that switching which branch is being served by manually editing the nf_ab cookie value and refreshing gives different results for a given Gatsby build/deploy. In some cases, refreshing the page seems to fix the issue, in others it does not. If I trigger a manual rebuild/deploy of both branches, the issue will sometimes go away completely, and in other cases it will change to the other branch. This is unexpected.
Basically, it seems like one of the JS files isn’t being connected or routed correctly for a given deploy, causing parts of the page which are JS reliant such as embedded videos and forms, not to load. We suspect this is somehow connected to split testing, but it is hard for us to trace back more than we already have as there are no errors in our deploy logs or consoles, and we can’t reproduce it locally.
I’ve attached several screenshots of the errors including request header info.
Any help would be greatly appreciated.
Package.json:
"scripts": {
"build": "gatsby build",
"clean": "gatsby clean",
"dev": "gatsby develop -o -H 0.0.0.0",
"develop": "gatsby develop",
"serve": "gatsby serve",
"lint": "eslint . --ext .js,.jsx",
"lint:fix": "eslint . --ext .js,.jsx --fix",
"lint:ci": "yarn lint --format junit -o results/eslint/result.xml",
"lint:staged": "eslint --fix --ext .js,.jsx",
"format": "prettier \"**/*.md \" --write",
"cy:open": "cypress open",
"cy:run": "cypress run",
"cy:run:ci": "cypress run --browser chrome --reporter junit --reporter-options 'mochaFile=results/cypress/result.xml'",
"prettier": "prettier",
"prettier:fix": "prettier --write \"**/*.{js,jsx}\"",
"test:e2e:dev": "cross-env CYPRESS_SUPPORT=y start-server-and-test dev http://localhost:8000 cy:open",
"test:e2e:run": "cross-env CYPRESS_SUPPORT=y start-server-and-test develop http://localhost:8000 cy:run",
"test:e2e:ci": "cross-env WAIT_ON_TIMEOUT=600000 CYPRESS_SUPPORT=y start-server-and-test develop http://localhost:8000 cy:run:ci",
"test": "jest"
},
"lint-staged": {
"*.{js,jsx}": [
"yarn prettier --write",
"yarn lint:staged"
],
"*.{yaml,yml}": [
"yarn prettier --write"
]
},
"dependencies": {
"@emotion/css": "^11.1.3",
"@emotion/react": "^11.0.0",
"@emotion/styled": "^11.0.0",
"@material-ui/core": "^4.12.3",
"@netlify/functions": "^0.11.0",
"@reach/skip-nav": "^0.16.0",
"@use-it/interval": "^1.0.0",
"@vimeo/player": "^2.16.3",
"airtable": "^0.11.1",
"aws-amplify": "^4.3.14",
"emotion": "^11.0.0",
"emotion-server": "^11.0.0",
"gatsby": "^4.7.1",
"gatsby-plugin-canonical-urls": "^4.7.0",
"gatsby-plugin-emotion": "^7.7.0",
"gatsby-plugin-image": "^2.7.0",
"gatsby-plugin-manifest": "^4.7.0",
"gatsby-plugin-material-ui": "^4.1.0",
"gatsby-plugin-prismic-previews": "^5.0.3",
"gatsby-plugin-react-helmet": "^5.7.0",
"gatsby-plugin-remove-console": "^0.0.2",
"gatsby-plugin-remove-serviceworker": "^1.0.0",
"gatsby-plugin-robots-txt": "^1.7.0",
"gatsby-plugin-sitemap": "^5.7.0",
"gatsby-source-prismic": "^5.1.0",
"get-contrast": "^3.0.0",
"idx": "^2.5.6",
"material-ui-popup-state": "^2.0.0",
"netlify-lambda": "^2.0.15",
"node-fetch": "^2.6.2",
"prismic-reactjs": "^1.3.4",
"react": "^17.0.2",
"react-colorful": "^5.5.1",
"react-dom": "^17.0.2",
"react-focus-trap": "^2.7.1",
"react-helmet": "^6.1.0",
"react-hubspot-form": "^1.3.7",
"react-lottie-player": "^1.4.0",
"react-phone-number-input": "^3.1.46",
"react-reveal": "^1.2.2",
"react-select": "^5.1.0",
"react-share": "^4.4.0"
},
"devDependencies": {
"@babel/eslint-parser": "^7.15.8",
"@netlify/plugin-gatsby": "^2.0.2",
"@testing-library/cypress": "^8.0.1",
"@testing-library/react": "^12.1.2",
"babel-jest": "^24.7.1",
"babel-plugin-transform-remove-console": "^6.9.4",
"babel-preset-gatsby": "^2.7.0",
"cross-env": "^7.0.3",
"cypress": "^8.6.0",
"eslint": "^7.32.0",
"eslint-config-airbnb": "^18.2.1",
"eslint-config-prettier": "^8.3.0",
"eslint-plugin-cypress": "^2.12.1",
"eslint-plugin-import": "^2.25.2",
"eslint-plugin-jsx-a11y": "^6.4.1",
"eslint-plugin-prettier": "^4.0.0",
"eslint-plugin-react": "^7.26.1",
"eslint-plugin-react-hooks": "^4.2.0",
"gatsby-cypress": "^1.14.0",
"gatsby-plugin-netlify": "^4.1.0",
"husky": "^7.0.2",
"identity-obj-proxy": "^3.0.0",
"jest": "^24.8.0",
"lint-staged": "^11.2.3",
"prettier": "^2.4.1",
"start-server-and-test": "^1.14.0"
},
"peerDependencies": {
"gatsby": "^4.0.0"
}
}
Feeling pretty disappointed and stuck as this is also a huge blocker for us. Really hope to hear something soon Netlify…
