Skip to content

Commit

Permalink
fixed youtube comments issue; fixed downloading issue;
Browse files Browse the repository at this point in the history
  • Loading branch information
@netptop
netptop committed May 19, 2020
1 parent bc491f5 commit 5818e3f
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 69 deletions.
92 changes: 48 additions & 44 deletions Proxy.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,18 @@ function countUtf8Bytes(s) {
return b
}

var contentTypeIsText = (headers) => {
if (!headers["content-type"] ||
headers["content-type"].indexOf('text/') !== -1 ||
headers["content-type"].indexOf('javascript') !== -1 ||
headers["content-type"].indexOf('urlencoded') !== -1 ||
headers["content-type"].indexOf('json') !== -1) {
return true
} else {
return false
}
}

var enableCors = function(req, res) {
if (req.headers['access-control-request-method']) {
res.setHeader('access-control-allow-methods', req.headers['access-control-request-method']);
Expand Down Expand Up @@ -101,7 +113,7 @@ let getHostFromReq = (req) => { //return target

let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace}) => {
let stream = fs.createWriteStream("web-records.csv", {flags:'a'})
let handleRespond = ({req, res, body, gbFlag}) => {
let handleRespond = ({req, res, body, gbFlag}) => { // text file
let myRe
let {host, httpType} = getHostFromReq(req)
let location = res.getHeaders()['location']
Expand Down Expand Up @@ -233,11 +245,18 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
}
bodyLength += data.length
bodyList.push(data)
if (res.getHeader('content-type') && res.getHeader('content-type').indexOf('video') !== -1) {
if (bodyLength >= 2500000 && contentTypeIsText(proxyRes.headers) !== true) {
let body = Buffer.concat(bodyList)
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
if ((host.indexOf('cdn') !== -1 && bodyLength >= 105000000) ||
(host.indexOf('cdn') === -1 && bodyLength >= 2500000)) {
let contentType = proxyRes.headers['content-type']
let contentLen = proxyRes.headers['content-length']
if (contentLen >= 155000000 ||
(host.indexOf('googlevideo') !== -1 && contentLen >= 2500000)) {
}
console.log(`route:${fwdStr}, content-type:${contentType},bulk length:${bodyLength}, content-length:${contentLen}, ${host}`)
bodyList = []
bodyLength = 0
res.write(body)
}
})
proxyRes.on('end', function() {
Expand All @@ -247,7 +266,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
let body = Buffer.concat(bodyList)
let gbFlag = false
if (proxyRes.headers["content-encoding"] === 'gzip' ||
proxyRes.headers["content-encoding"] === 'br') {
proxyRes.headers["content-encoding"] === 'br') { // gzip/br encoding
let gunzipped
try {
if (proxyRes.headers["content-encoding"] === 'br') {
Expand All @@ -261,11 +280,7 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
// res.status(404).send(`{"error": "${e}"}`)
return
}
if (!proxyRes.headers["content-type"] ||
proxyRes.headers["content-type"].indexOf('text/') !== -1 ||
proxyRes.headers["content-type"].indexOf('javascript') !== -1 ||
proxyRes.headers["content-type"].indexOf('urlencoded') !== -1 ||
proxyRes.headers["content-type"].indexOf('json') !== -1) {
if (contentTypeIsText(proxyRes.headers) === true) { //gzip and text
if (!gunzipped) {
// res.status(404).send(`{"error":"failed unzip"}`)
redirect2HomePage({res, httpprefix, serverName,})
Expand All @@ -292,29 +307,20 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
res.setHeader('service-worker-allowed','/')
}
handleRespond({req, res, body, gbFlag})
} else {
} else { // gzip and non-text
// console.log(`2========>${logGet()}`)
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
let contentType = proxyRes.headers['content-type']
let contentLen = proxyRes.headers['content-length']
console.log(`end,route:${fwdStr}, content-type:${contentType},gzip length:${bodyLength}, content-length:${contentLen}, ${host}`)
try {
let key = "content-encoding"
if(key in proxyRes.headers) {
res.setHeader(key, proxyRes.headers[key]);
}
logSave(`2: res.headers:${JSON.stringify(res.getHeaders())}`)
if (req.headers['debugflag']==='true') {
res.removeHeader('content-encoding')
res.setHeader('content-type','text/plain')
body=logGet()
}
res.end(body)
} catch(e) {
console.log(`error:${e}`)
}
}
} else if (proxyRes.statusCode === 301 || proxyRes.statusCode === 302 || proxyRes.statusCode === 307 || proxyRes.statusCode === 308 ||
(proxyRes.headers["content-type"] &&
(proxyRes.headers["content-type"].indexOf('text/') !== -1 ||
proxyRes.headers["content-type"].indexOf('javascript') !== -1 ||
proxyRes.headers["content-type"].indexOf('json') !== -1))) {
contentTypeIsText(proxyRes.headers) === true) { // text with non gzip encoding
logSave(`utf-8 text...`)
let originBody = body
body = body.toString('utf-8');
Expand All @@ -326,17 +332,12 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
gbFlag = true
}
handleRespond({req, res, body, gbFlag})
} else {
} else { // non-gzip and non-text body
logSave(`3========>${logGet()}`)
if (req.headers['debugflag']==='true') {
res.removeHeader('content-encoding')
res.setHeader('content-type','text/plain')
body=logGet()
}
if (res.getHeader('content-type') && res.getHeader('content-type').indexOf('video') !== -1) {
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
console.log(`route:${fwdStr}, length:${bodyLength}, ${host}`)
}
let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']
let contentType = proxyRes.headers['content-type']
let contentLen = proxyRes.headers['content-length']
console.log(`end,route:${fwdStr}, content-type:${contentType},length:${bodyLength}, content-length:${contentLen}, ${host}`)
res.end(body)
}
})
Expand All @@ -362,18 +363,18 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
.map(cookie => cookie.toString())
proxyRes.headers['set-cookie'] = modifiedSetCookieHeaders
Object.keys(proxyRes.headers).forEach(function (key) {
if (key === 'content-encoding' ||
key === 'content-security-policy' ||
if (key === 'content-security-policy' ||
key === 'x-frame-options' ||
(key === 'content-length' && proxyRes.headers["content-type"] &&
(proxyRes.headers["content-type"].indexOf('text/') !== -1 ||
proxyRes.headers["content-type"].indexOf('javascript') !== -1 ||
proxyRes.headers["content-type"].indexOf('json') !== -1))) {
(key === 'content-length' && contentTypeIsText(proxyRes.headers) === true)) {
logSave(`skip header:${key}`)
return
}
try {
res.setHeader(key, proxyRes.headers[key]);
if (key === 'content-encoding' && contentTypeIsText(proxyRes.headers) === true) {
res.setHeader(key, 'gzip') // for text response, we need to set it gzip encoding cuz we will do gzip on it
} else {
res.setHeader(key, proxyRes.headers[key])
}
} catch(e) {
logSave(`error:${e}`)
return
Expand All @@ -392,8 +393,11 @@ let Proxy = ({blockedSites, urlModify, httpprefix, serverName, port, cookieDomai
}
},
onProxyReq: (proxyReq, req, res) => {
let myRe = new RegExp(`/${httpprefix}/${serverName}.*?/`, 'g') // match group
req.url = req.url.replace(myRe, '/')
let myRe = new RegExp(`/http[s]?/${serverName}[0-9:]*?`, 'g') // match group
req.url = req.url.replace(myRe, '')
if (req.url.length === 0) {
req.url = '/'
}

let fwdStr = req.headers['X-Forwarded-For'] || req.headers['x-forwarded-for']

Expand Down
3 changes: 2 additions & 1 deletion config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ let config = {
httpprefix: 'https', port: 443,
serverName: 'proxy.netptop.com',
}
let blockedSites = []
let blockedSites = ['www.chase.com'] // accessing to chase.com was reported by google

if (process.env.herokuAddr) {
config.serverName = process.env.herokuAddr
Expand Down Expand Up @@ -177,6 +177,7 @@ const siteSpecificReplace = {
// '(.\..=this\.fx[(][)]);return (.)': `$1;$2.bandwidthEstimate=1000.1;return $2`,// a.C=this.fx();return a
'[a-zA-Z]\.setSizeStyle[(]..,.[)]': `1`,
'a\.....\.style.display=0===.."none":"";': `;`, // a.A[c].style.display = 0 === b ? "none" : "";
'="/(watch_fragments2_ajax)"': `="/https/www.youtube.com/$1"`,
},
'm.youtube.com': {
'"/(results.search_query=)': `"/https/m.youtube.com/$1`,
Expand Down
13 changes: 9 additions & 4 deletions index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,7 @@ <h1 class="logo">
</a>
</div>
<div class="col-md-3 col-sm-4 col-xs-6">
<a href="/https/www.youtube.com/playlist?list=PLPVsJMpVPkCd_OUBs7Aj_dYEbwgO8ZtLY" class="card-link" target="_blank">
<a href="/https/www.youtube.com/channel/UCAq_xQV8pJ2Q_KOszzaYPBg/feed?disable_polymer=1" class="card-link" target="_blank">
<div class="card-tit">YouTube</div>
</a>
</div>
Expand All @@ -793,7 +793,7 @@ <h1 class="logo">
</a>
</div>
<div class="col-md-3 col-sm-4 col-xs-6">
<a href="https://github.com/esu-tg/tg-app/blob/master/README.md" class="card-link" target="_blank">
<a href="/https/telegram.org/dl/android/apk" class="card-link" target="_blank">
<div class="card-tit">Telegram下载</div>
</a>
</div>
Expand Down Expand Up @@ -860,8 +860,13 @@ <h1 class="logo">
</a>
</div>
<div class="col-md-3 col-sm-4 col-xs-6">
<a href="/https/www.zaobao.com.sg/" class="card-link" target="_blank">
<div class="card-tit">新加坡联合早报</div>
<a href="/http/cnnews.chosun.com/" class="card-link" target="_blank">
<div class="card-tit">朝鲜日报</div>
</a>
</div>
<div class="col-md-3 col-sm-4 col-xs-6">
<a href="/https/std.stheadline.com/" class="card-link" target="_blank">
<div class="card-tit">星岛日报</div>
</a>
</div>
</div>
Expand Down
11 changes: 7 additions & 4 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@ let cookieDomainRewrite = serverName
let proxy = Proxy({ blockedSites, urlModify, httpprefix, serverName, port, cookieDomainRewrite, locationReplaceMap302, regReplaceMap, siteSpecificReplace, pathReplace})

const middle1 = (req, res, next) => {
let timestr = new Date().toISOString()
let myRe = new RegExp(`/http[s]?/${serverName}.*?/`, 'g') // match group // remove duplicate https/${serverName}:${port}/
req.url = req.url.replace(myRe, '/')
let timestr = new Date().toISOString()
let myRe = new RegExp(`/http[s]?/${serverName}[0-9:]*?`, 'g') // match group
req.url = req.url.replace(myRe, '')
if (req.url.length === 0) {
req.url = '/'
}

console.log(`${timestr}: req.url:${req.url}`)
console.log(`${timestr}: req.url:${req.url}`)
const dirPath = path.join(__dirname, req.url)
let fwdStr = req.headers['x-forwarded-for']
if (fwdStr && fwdStr.split(',').length > 3) { // too many forwardings
Expand Down
16 changes: 0 additions & 16 deletions test/siteproxylocal.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,22 +240,6 @@ test('youtube homepage issue', async () => {
// ht-time":"0","x-request-start":"1584141848398","total-route-time":"0"}
// h2020-03-13T23:24:08.403097+00:00 app[web.1]: httpType:https, host:id.google.com

test('no content-type field situation', async () => {
const url = `${httpprefix}://${serverName}:${port}/https/onetag-sys.com/usync/?pubId=5927d926323dc2c`
const response = await axios({
method: 'get',
headers: {
'Accept': `text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9`,
},
maxRedirects: 0,
validateStatus: null, // important for status 302
url,
})
// console.log(`${JSON.stringify(response.headers)}`)
// console.log(`${response.data}`)
expect(typeof(response.data)).toBe('object')
}, 15000); // should be done within 3 seconds.

test('doubi issue', async () => {
const url = `${httpprefix}://${serverName}:${port}/https/doubibackup.com`
const response = await axios({
Expand Down

0 comments on commit 5818e3f

Please sign in to comment.