fixed gbk issue; removed the user-agent string for youtube

Proxy.js

@@ -367,7 +367,8 @@ let Proxy = ({ProxyMiddleware, blockedSites, urlModify, httpprefix, serverName,
                 }
                 let searchBody = body.slice(0, 1000)
                 if (searchBody.indexOf('="text/html; charset=gb') !== -1 ||
-                    searchBody.indexOf(' charset="gb') !== -1 ||
+                    searchBody.search(/ontent=.*charset="gb/) !== -1 ||
+                    searchBody.search(/ONTENT=.*charset="gb/) !== -1 ||
                     searchBody.indexOf('=\'text/html; charset=gb') !== -1) {
                     logSave(`gb2312 found...`)
                     body = iconv.decode(originBody, 'gbk')
@@ -551,7 +552,7 @@ let Proxy = ({ProxyMiddleware, blockedSites, urlModify, httpprefix, serverName,
         if (host.indexOf('youtube.com') !== -1) {
             // proxyReq.setHeader('User-Agent', `Opera/7.50 (Windows XP; U)`)
             // proxyReq.setHeader('User-Agent', `Opera/9.80 (Android 4.1.2; Linux; Opera Mobi/ADR-1305251841) Presto/2.11.355 Version/12.10`)
-            proxyReq.setHeader('User-Agent', `Mozilla/5.0 (Linux; Android 6.0.1; Lenovo-A6020l36 Build/MMB29M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.93 Mobile Safari/537.36`)
+            // proxyReq.setHeader('User-Agent', `Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0`)
         }
         logSave(`req host:${host}, req.url:${req.url}, proxyReq.query:${proxyReq.query} proxyReq.path:${proxyReq.path}, proxyReq.url:${proxyReq.url} proxyReq headers:${JSON.stringify(proxyReq.getHeaders())}`)
         if(host === '' || !host) {

package.json

@@ -8,6 +8,7 @@
     "test": "jest",
     "localtest": "localFlag=true jest siteproxylocal.test.js",
     "cftest": "jest cloudflare.test.js",
+    "herokutest": "jest herokuapp.test.js",
     "start": "node --tls-min-v1.0 index.js",
     "localstartNow": "localFlag=true node --tls-min-v1.0 index.js",
     "localstart": "PORT=8011 localFlag=true node --tls-min-v1.0 index.js"
@@ -19,7 +20,7 @@
     "cookiejar": "^2.1.2",
     "express": "^4.17.1",
     "http-proxy-middleware": "^0.20.0",
-    "iconv-lite": "^0.5.1",
+    "iconv-lite": "^0.6.2",
     "pako": "^1.0.11",
     "query-string": "^6.11.1",
     "url-parse": "^1.4.7"

test/herokuapp.test.js

@@ -0,0 +1,34 @@
+/**
+ * @jest-environment node
+ */
+const axios = require('axios');
+
+const httpprefix = 'https'
+const serverName = 'siteproxy.herokuapp.com'
+const port = '443'
+
+test('post operation', async () => {
+  const url = `${httpprefix}://${serverName}:${port}/https/api.twitter.com/1.1/branch/init.json`
+  const response = await axios({
+    method: 'post',
+    validateStatus: null, // important for status 302/403
+    headers: {
+      authorization: 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+      connection: 'keep-alive',
+      cookie: 'personalization_id="v1_DkZdb/rVpgZVDtiT3qVAUA=="; guest_id=v1%3A160056087565376493; _ga=GA1.3.1320916387.1600483268; _gid=GA1.3.1182741545.1600483268; ORIGINALHOST=https/api.twitter.com',
+      Host: 'siteproxy.herokuapp.com',
+      Referer: 'https://siteproxy.herokuapp.com/explore',
+      'Sec-Fetch-Dest': 'empty',
+      'Sec-Fetch-Mode': 'cors',
+      'Sec-Fetch-Site': 'same-origin',
+      'x-csrf-token': 'ac1aa9d58c8798f0932410a1a564eb42',
+      'x-twitter-active-user': 'yes',
+      'x-twitter-client-language': 'en',
+    },
+    url,
+    data:'',
+  })
+  console.log(`${JSON.stringify(response.headers)}`)
+  console.log(`${JSON.stringify(response.data)}`)
+  // expect(JSON.stringify(response.data).indexOf(`test data123`)).not.toBe(-1)
+}, 15000); // should be done within 3 seconds.