Skip to content

Commit 5818e3f

Browse files
netptopnetptop
committed
fixed youtube comments issue; fixed downloading issue;
1 parent bc491f5 commit 5818e3f

File tree

5 files changed

+66
-69
lines changed

5 files changed

+66
-69
lines changed

‎Proxy.js

+48-44
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,18 @@ function countUtf8Bytes(s) {
1414
return b
1515
}
1616

17+
var contentTypeIsText = (headers) => {
18+
if (!headers["content-type"] ||
19+
headers["content-type"].indexOf('text/') !== -1 ||
20+
headers["content-type"].indexOf('javascript') !== -1 ||
21+
headers["content-type"].indexOf('urlencoded') !== -1 ||
22+
headers["content-type"].indexOf('json') !== -1) {
23+
return true
24+
} else {
25+
return false
26+
}
27+
}
28+
1729
var enableCors = function(req, res) {
1830
if (req.headers['access-control-request-method']) {
1931
res.setHeader('access-control-allow-methods', req.headers['access-control-request-method']);
@@ -101,7 +113,7 @@ let getHostFromReq = (req) => { //return target
101113

102114
let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace}) => {
103115
let stream = fs.createWriteStream("web-records.csv", {flags:'a'})
104-
let handleRespond = ({req, res, body, gbFlag}) => {
116+
let handleRespond = ({req, res, body, gbFlag}) => { // text file
105117
let myRe
106118
let {host, httpType} = getHostFromReq(req)
107119
let location = res.getHeaders()['location']
@@ -233,11 +245,18 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
233245
}
234246
bodyLength += data.length
235247
bodyList.push(data)
236-
if (res.getHeader('content-type') && res.getHeader('content-type').indexOf('video') !== -1) {
248+
if (bodyLength >= 2500000 && contentTypeIsText(proxyRes.headers) !== true) {
249+
let body = Buffer.concat(bodyList)
237250
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
238-
if ((host.indexOf('cdn') !== -1 && bodyLength >= 105000000) ||
239-
(host.indexOf('cdn') === -1 && bodyLength >= 2500000)) {
251+
let contentType = proxyRes.headers['content-type']
252+
let contentLen = proxyRes.headers['content-length']
253+
if (contentLen >= 155000000 ||
254+
(host.indexOf('googlevideo') !== -1 && contentLen >= 2500000)) {
240255
}
256+
console.log(`route:${fwdStr}, content-type:${contentType},bulk length:${bodyLength}, content-length:${contentLen}, ${host}`)
257+
bodyList = []
258+
bodyLength = 0
259+
res.write(body)
241260
}
242261
})
243262
proxyRes.on('end', function() {
@@ -247,7 +266,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
247266
let body = Buffer.concat(bodyList)
248267
let gbFlag = false
249268
if (proxyRes.headers["content-encoding"] === 'gzip' ||
250-
proxyRes.headers["content-encoding"] === 'br') {
269+
proxyRes.headers["content-encoding"] === 'br') { // gzip/br encoding
251270
let gunzipped
252271
try {
253272
if (proxyRes.headers["content-encoding"] === 'br') {
@@ -261,11 +280,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
261280
// res.status(404).send(`{"error": "${e}"}`)
262281
return
263282
}
264-
if (!proxyRes.headers["content-type"] ||
265-
proxyRes.headers["content-type"].indexOf('text/') !== -1 ||
266-
proxyRes.headers["content-type"].indexOf('javascript') !== -1 ||
267-
proxyRes.headers["content-type"].indexOf('urlencoded') !== -1 ||
268-
proxyRes.headers["content-type"].indexOf('json') !== -1) {
283+
if (contentTypeIsText(proxyRes.headers) === true) { //gzip and text
269284
if (!gunzipped) {
270285
// res.status(404).send(`{"error":"failed unzip"}`)
271286
redirect2HomePage({res, httpprefix, serverName,})
@@ -292,29 +307,20 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
292307
res.setHeader('service-worker-allowed','/')
293308
}
294309
handleRespond({req, res, body, gbFlag})
295-
} else {
310+
} else { // gzip and non-text
296311
// console.log(`2========>${logGet()}`)
312+
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
313+
let contentType = proxyRes.headers['content-type']
314+
let contentLen = proxyRes.headers['content-length']
315+
console.log(`end,route:${fwdStr}, content-type:${contentType},gzip length:${bodyLength}, content-length:${contentLen}, ${host}`)
297316
try {
298-
let key = "content-encoding"
299-
if(key in proxyRes.headers) {
300-
res.setHeader(key, proxyRes.headers[key]);
301-
}
302-
logSave(`2: res.headers:${JSON.stringify(res.getHeaders())}`)
303-
if (req.headers['debugflag']==='true') {
304-
res.removeHeader('content-encoding')
305-
res.setHeader('content-type','text/plain')
306-
body=logGet()
307-
}
308317
res.end(body)
309318
} catch(e) {
310319
console.log(`error:${e}`)
311320
}
312321
}
313322
} else if (proxyRes.statusCode === 301 || proxyRes.statusCode === 302 || proxyRes.statusCode === 307 || proxyRes.statusCode === 308 ||
314-
(proxyRes.headers["content-type"] &&
315-
(proxyRes.headers["content-type"].indexOf('text/') !== -1 ||
316-
proxyRes.headers["content-type"].indexOf('javascript') !== -1 ||
317-
proxyRes.headers["content-type"].indexOf('json') !== -1))) {
323+
contentTypeIsText(proxyRes.headers) === true) { // text with non gzip encoding
318324
logSave(`utf-8 text...`)
319325
let originBody = body
320326
body = body.toString('utf-8');
@@ -326,17 +332,12 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
326332
gbFlag = true
327333
}
328334
handleRespond({req, res, body, gbFlag})
329-
} else {
335+
} else { // non-gzip and non-text body
330336
logSave(`3========>${logGet()}`)
331-
if (req.headers['debugflag']==='true') {
332-
res.removeHeader('content-encoding')
333-
res.setHeader('content-type','text/plain')
334-
body=logGet()
335-
}
336-
if (res.getHeader('content-type') && res.getHeader('content-type').indexOf('video') !== -1) {
337-
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
338-
console.log(`route:${fwdStr}, length:${bodyLength}, ${host}`)
339-
}
337+
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
338+
let contentType = proxyRes.headers['content-type']
339+
let contentLen = proxyRes.headers['content-length']
340+
console.log(`end,route:${fwdStr}, content-type:${contentType},length:${bodyLength}, content-length:${contentLen}, ${host}`)
340341
res.end(body)
341342
}
342343
})
@@ -362,18 +363,18 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
362363
.map(cookie => cookie.toString())
363364
proxyRes.headers['set-cookie'] = modifiedSetCookieHeaders
364365
Object.keys(proxyRes.headers).forEach(function (key) {
365-
if (key === 'content-encoding' ||
366-
key === 'content-security-policy' ||
366+
if (key === 'content-security-policy' ||
367367
key === 'x-frame-options' ||
368-
(key === 'content-length' && proxyRes.headers["content-type"] &&
369-
(proxyRes.headers["content-type"].indexOf('text/') !== -1 ||
370-
proxyRes.headers["content-type"].indexOf('javascript') !== -1 ||
371-
proxyRes.headers["content-type"].indexOf('json') !== -1))) {
368+
(key === 'content-length' && contentTypeIsText(proxyRes.headers) === true)) {
372369
logSave(`skip header:${key}`)
373370
return
374371
}
375372
try {
376-
res.setHeader(key, proxyRes.headers[key]);
373+
if (key === 'content-encoding' && contentTypeIsText(proxyRes.headers) === true) {
374+
res.setHeader(key, 'gzip') // for text response, we need to set it gzip encoding cuz we will do gzip on it
375+
} else {
376+
res.setHeader(key, proxyRes.headers[key])
377+
}
377378
} catch(e) {
378379
logSave(`error:${e}`)
379380
return
@@ -392,8 +393,11 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
392393
}
393394
},
394395
onProxyReq: (proxyReq, req, res) => {
395-
let myRe = new RegExp(`/${httpprefix}/${serverName}.*?/`, 'g') // match group
396-
req.url = req.url.replace(myRe, '/')
396+
let myRe = new RegExp(`/http[s]?/${serverName}[0-9:]*?`, 'g') // match group
397+
req.url = req.url.replace(myRe, '')
398+
if (req.url.length === 0) {
399+
req.url = '/'
400+
}
397401

398402
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
399403

‎config.js

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ let config = {
1111
httpprefix: 'https', port: 443,
1212
serverName: 'proxy.netptop.com',
1313
}
14-
let blockedSites = []
14+
let blockedSites = ['www.chase.com'] // accessing to chase.com was reported by google
1515

1616
if (process.env.herokuAddr) {
1717
config.serverName = process.env.herokuAddr
@@ -177,6 +177,7 @@ const siteSpecificReplace = {
177177
// '(.\..=this\.fx[(][)]);return (.)': `$1;$2.bandwidthEstimate=1000.1;return $2`,// a.C=this.fx();return a
178178
'[a-zA-Z]\.setSizeStyle[(]..,.[)]': `1`,
179179
'a\.....\.style.display=0===.."none":"";': `;`, // a.A[c].style.display = 0 === b ? "none" : "";
180+
'="/(watch_fragments2_ajax)"': `="/https/www.youtube.com/$1"`,
180181
},
181182
'm.youtube.com': {
182183
'"/(results.search_query=)': `"/https/m.youtube.com/$1`,

‎index.html

+9-4
Original file line numberDiff line numberDiff line change
@@ -778,7 +778,7 @@ <h1 class="logo">
778778
</a>
779779
</div>
780780
<div class="col-md-3 col-sm-4 col-xs-6">
781-
<a href="/https/www.youtube.com/playlist?list=PLPVsJMpVPkCd_OUBs7Aj_dYEbwgO8ZtLY" class="card-link" target="_blank">
781+
<a href="/https/www.youtube.com/channel/UCAq_xQV8pJ2Q_KOszzaYPBg/feed?disable_polymer=1" class="card-link" target="_blank">
782782
<div class="card-tit">YouTube</div>
783783
</a>
784784
</div>
@@ -793,7 +793,7 @@ <h1 class="logo">
793793
</a>
794794
</div>
795795
<div class="col-md-3 col-sm-4 col-xs-6">
796-
<a href="https://github.com/esu-tg/tg-app/blob/master/README.md" class="card-link" target="_blank">
796+
<a href="/https/telegram.org/dl/android/apk" class="card-link" target="_blank">
797797
<div class="card-tit">Telegram下载</div>
798798
</a>
799799
</div>
@@ -860,8 +860,13 @@ <h1 class="logo">
860860
</a>
861861
</div>
862862
<div class="col-md-3 col-sm-4 col-xs-6">
863-
<a href="/https/www.zaobao.com.sg/" class="card-link" target="_blank">
864-
<div class="card-tit">新加坡联合早报</div>
863+
<a href="/http/cnnews.chosun.com/" class="card-link" target="_blank">
864+
<div class="card-tit">朝鲜日报</div>
865+
</a>
866+
</div>
867+
<div class="col-md-3 col-sm-4 col-xs-6">
868+
<a href="/https/std.stheadline.com/" class="card-link" target="_blank">
869+
<div class="card-tit">星岛日报</div>
865870
</a>
866871
</div>
867872
</div>

‎index.js

+7-4
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,14 @@ let cookieDomainRewrite = serverName
99
let proxy = Proxy({ blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace})
1010

1111
const middle1 = (req, res, next) => {
12-
let timestr = new Date().toISOString()
13-
let myRe = new RegExp(`/http[s]?/${serverName}.*?/`, 'g') // match group // remove duplicate https/${serverName}:${port}/
14-
req.url = req.url.replace(myRe, '/')
12+
let timestr = new Date().toISOString()
13+
let myRe = new RegExp(`/http[s]?/${serverName}[0-9:]*?`, 'g') // match group
14+
req.url = req.url.replace(myRe, '')
15+
if (req.url.length === 0) {
16+
req.url = '/'
17+
}
1518

16-
console.log(`${timestr}: req.url:${req.url}`)
19+
console.log(`${timestr}: req.url:${req.url}`)
1720
const dirPath = path.join(__dirname, req.url)
1821
let fwdStr = req.headers['x-forwarded-for']
1922
if (fwdStr && fwdStr.split(',').length > 3) { // too many forwardings

‎test/siteproxylocal.test.js

-16
Original file line numberDiff line numberDiff line change
@@ -240,22 +240,6 @@ test('youtube homepage issue', async () => {
240240
// ht-time":"0","x-request-start":"1584141848398","total-route-time":"0"}
241241
// h2020-03-13T23:24:08.403097+00:00 app[web.1]: httpType:https, host:id.google.com
242242

243-
test('no content-type field situation', async () => {
244-
const url = `${httpprefix}://${serverName}:${port}/https/onetag-sys.com/usync/?pubId=5927d926323dc2c`
245-
const response = await axios({
246-
method: 'get',
247-
headers: {
248-
'Accept': `text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9`,
249-
},
250-
maxRedirects: 0,
251-
validateStatus: null, // important for status 302
252-
url,
253-
})
254-
// console.log(`${JSON.stringify(response.headers)}`)
255-
// console.log(`${response.data}`)
256-
expect(typeof(response.data)).toBe('object')
257-
}, 15000); // should be done within 3 seconds.
258-
259243
test('doubi issue', async () => {
260244
const url = `${httpprefix}://${serverName}:${port}/https/doubibackup.com`
261245
const response = await axios({

0 commit comments

Comments
 (0)