diff --git a/README.md b/README.md index 741a5081b9952..8c1234f977df6 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@
-[![logo](src/img/logo-b.webp)](https://github.com/ipitio/backage) +[![logo](src/img/logo-b.webp)](https://github.com/pkgforge-dev/backage) -# [backage](https://github.com/ipitio/backage) +# [backage](https://github.com/pkgforge-dev/backage) **It's all part and parcel** --- -[![packages](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fipitio%2Fbackage%2Fraw%2Findex%2F.json&query=%24.packages&logo=github&logoColor=959da5&label=packages&labelColor=333a41&color=grey)](https://github.com/ipitio/backage/tree/index) [![updated](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fipitio%2Fbackage%2Fraw%2Findex%2F.json&query=%24.date&logo=github&logoColor=959da5&label=updated&labelColor=333a41&color=indigo)](https://github.com/ipitio/backage/releases/latest) +[![packages](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fpkgforge-dev%2Fbackage%2Fraw%2Findex%2F.json&query=%24.packages&logo=github&logoColor=959da5&label=packages&labelColor=333a41&color=grey)](https://github.com/pkgforge-dev/backage/tree/index) [![updated](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fpkgforge-dev%2Fbackage%2Fraw%2Findex%2F.json&query=%24.date&logo=github&logoColor=959da5&label=updated&labelColor=333a41&color=indigo)](https://github.com/pkgforge-dev/backage/releases/latest) -[![tag](https://img.shields.io/badge/dynamic/xml?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fipitio%2Fbackage%2Fraw%2Findex%2Fipitio%2Fbackage%2Fbackage.xml&query=%2Fxml%2Fversion%5B.%2Flatest%5B.%3D%22true%22%5D%5D%2Ftags%5B.!%3D%22latest%22%5D&logo=github&logoColor=959da5&label=tag&labelColor=333a41&color=black)](https://github.com/ipitio/backage/pkgs/container/backage) [![build](https://github.com/ipitio/backage/actions/workflows/publish.yml/badge.svg)](https://github.com/ipitio/backage/pkgs/container/backage) [![size](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fipitio%2Fbackage%2Fraw%2Findex%2Fipitio%2Fbackage%2Fbackage.json&query=%24.size&logo=github&logoColor=959da5&label=size&labelColor=333a41&color=sienna)](https://github.com/ipitio/backage/pkgs/container/backage) [![runs](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fipitio%2Fbackage%2Fraw%2Findex%2Fipitio%2Fbackage%2Fbackage.json&query=%24.downloads&logo=github&logoColor=959da5&label=runs&labelColor=333a41)](https://github.com/ipitio/backage/pkgs/container/backage) +[![tag](https://img.shields.io/badge/dynamic/xml?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fpkgforge-dev%2Fbackage%2Fraw%2Findex%2Fpkgforge-dev%2Fbackage%2Fbackage.xml&query=%2Fxml%2Fversion%5B.%2Flatest%5B.%3D%22true%22%5D%5D%2Ftags%5B.!%3D%22latest%22%5D&logo=github&logoColor=959da5&label=tag&labelColor=333a41&color=black)](https://github.com/pkgforge-dev/backage/pkgs/container/backage) [![build](https://github.com/pkgforge-dev/backage/actions/workflows/publish.yml/badge.svg)](https://github.com/pkgforge-dev/backage/pkgs/container/backage) [![size](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fpkgforge-dev%2Fbackage%2Fraw%2Findex%2Fpkgforge-dev%2Fbackage%2Fbackage.json&query=%24.size&logo=github&logoColor=959da5&label=size&labelColor=333a41&color=sienna)](https://github.com/pkgforge-dev/backage/pkgs/container/backage) [![runs](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fround-lake.dustinice.workers.dev%3A443%2Fhttps%2Fgithub.com%2Fpkgforge-dev%2Fbackage%2Fraw%2Findex%2Fpkgforge-dev%2Fbackage%2Fbackage.json&query=%24.downloads&logo=github&logoColor=959da5&label=runs&labelColor=333a41)](https://github.com/pkgforge-dev/backage/pkgs/container/backage)
@@ -18,15 +18,15 @@ Ever wish you could show npm, gem, mvn, Gradle, NuGet, or GHCR badges for GitHub ## Getting Started -If this is [`ipitio/backage`](https://github.com/ipitio/backage), all you have to do is **star the repo to get your public packages added!** The service's circular priority queue will update the [closed-loop system](https://github.com/ipitio/backage/releases/latest) with them within the next few hours. Additionally watching and forking the repo, and following the owner, are ways to increase their priority. Yes, I know, but these are the graphs GitHub has available. +If this is [`ipitio/backage`](https://github.com/ipitio/backage), all you have to do is **star the repo to get your public packages added!** The service's circular priority queue will update the [closed-loop system](https://github.com/pkgforge-dev/backage/releases/latest) with them within the next few hours. Additionally watching and forking the repo, and following the owner, are ways to increase their priority. Yes, I know, but these are the graphs GitHub has available. > [!WARNING] -> For this to work, your profile must be set to [public](https://github.com/ipitio/backage/issues/34#issuecomment-2968850773). +> Ensure your profile is [public](https://github.com/ipitio/backage/issues/34#issuecomment-2968850773) so that this repo can see your packages. -Otherwise, if this is a fork, you'd prefer an alternative method, or your packages weren't added to the [index](https://github.com/ipitio/backage/tree/index) after a day, enter the case-sensitive name of each missing user or organization on a new line at the top of `owners.txt` [here](https://github.com/ipitio/backage/edit/master/owners.txt) and make a pull request. Don't worry -- while my Contribution Graph is an uptime monitor of sorts, yours won't be. See the top of `bkg.sh` for details about available options, which must come last when passed to `update.sh`, as shown in `Self-Host`. +Otherwise, if this is a fork, you'd prefer an alternative method, or your packages weren't added to the [index](https://github.com/pkgforge-dev/backage/tree/index) after a day, enter the case-sensitive name of each missing user or organization on a new line at the top of the queue, `owners.txt`, [here](https://github.com/pkgforge-dev/backage/edit/master/owners.txt) and make a pull request. Don't worry -- while my Contribution Graph is an uptime monitor of sorts, yours won't be. See the top of `bkg.sh` for details about available options, which must come last when passed to `update.sh`, as shown in `Self-Host` below. > [!TIP] -> You only need to add the name(s), IDs are fetched as needed. +> You only need to add names to the queue; IDs are fetched as needed and entries are removed once processed. New packages won't be added until *all* existing ones are refreshed; you should also create an independent instance that'll update faster and more frequently. Simply fork just the `master` branch, choose one of the following options, and use the [Alternative URL](#alternative-url) when it changes. This centralized repo will then serve as a backup for all subsets of packages not in `optout.txt`. @@ -49,10 +49,10 @@ This will use a lot of minutes on GitHub-hosted runners, so you may want to use This is an example for `systemd`; adapt it to your needs. Please note: - Docker needs to be installed -- You don't need to set `GITHUB_TOKEN` if you're logged in with `gh` or you'll first use your PAT to run (replace `*` with `https` or `ssh`): +- You don't need to set `GITHUB_TOKEN` if you first log in with `gh` or use your PAT to run: ```bash -git clone --depth=1 -b master --single-branch *://@github.com/ipitio/backage /opt/backage/master +git clone --depth=1 -b master --single-branch [https|ssh]://@github.com/pkgforge-dev/backage /opt/backage/master ``` - `-m 0` ensures only the public packages of the owners you've added are updated (default) @@ -205,27 +205,27 @@ $.[FILTER].PROPERTY You can query a package for its properties, like size or version: ```py -/xml/PROPERTY +//PROPERTY ``` ```py -/xml/size +//size ``` Versions can be filtered in and tags out: ```py -/xml/version[FILTER]/PROPERTY +//version[FILTER]/PROPERTY ``` ```py -/xml/version[./latest[.="true"]]/tags[.!="latest"] +//version[./latest[.="true"]]/tags[.!="latest"] ``` As can packages in `owner[/repo]/.xml` files: ```py -/xml/package[FILTER]/PROPERTY +//package[FILTER]/PROPERTY ``` @@ -233,17 +233,17 @@ As can packages in `owner[/repo]/.xml` files: ### Alternative URL ```py -https://github.com/ipitio/backage/raw/index/OWNER/[REPO/[PACKAGE]].FORMAT +https://github.com/pkgforge-dev/backage/raw/index/OWNER/[REPO/[PACKAGE]].FORMAT ``` The endpoint is also available here! This will change to your fork once it updates. -## JSON2XML Proxy +## Bring Your Own JSON ```py https://ipitio.github.io/backage?json=https://URL/ENCODED/JSON ``` -Use your own external JSON with this proxy to convert it into XML. This doesn't currently work with Shields. Try it out in your browser: +While this doesn't directly work with Shields, you can BYOJ to convert into XML. Try it out in your browser: -**** +**** diff --git a/optout.txt b/optout.txt index 75c20e8db742b..e69de29bb2d1d 100644 --- a/optout.txt +++ b/optout.txt @@ -1,5 +0,0 @@ -airtower-luna/hello-ghcr/hello-ghcr-meow -hugomods/docker -nuxratx -pkgforge -pkgforge-dev diff --git a/owners.txt b/owners.txt index 74eedf656949f..e69de29bb2d1d 100644 --- a/owners.txt +++ b/owners.txt @@ -1,740 +0,0 @@ -1396634/taikoo -1396684/takahara -1396746/tamu-vscl -1396605/terranb -1398167/thecodeag -1396702/tonny-zhang -1396651/tonywongk -1397571/trilbymail -1396643/trpowell -1397679/uniplaces -1397698/upfrontsoftware -1396653/vinhto -1396659/viscerah -1396606/wangpeng468 -1397911/warthog-logic -1398444/weareid -1398743/webelectric -1396609/wenjiezhou -1397478/www2012 -1396629/xiaocoder -1396687/xkilluax -1396637/yeahugo -1396668/ypupo2002 -1397206/yujinrobot -1396642/zhangf01 -1401138/01022012 -1403226/2mxdev -1406201/2ndrategeniuses -1406734/ACUHO-I -1408783/ACM-UConn -1410514/ACMCMU -1413128/3DGTL -1414484/0xC3 -1414517/10kwa -1417435/ABRHS-Video-Game-Development-Club -1418513/AURIN -1420540/100yrs -1422710/22seeds -1422719/Abhay90 -1423543/ActiveStep -1423580/81-50-3631-0637-TO-TALK -1423659/Alex-ZI -1425390/Ayushorg -1426568/3111H-Focus -1428461/ABAgile -1430388/AP-CS -1430484/14100 -1430498/8pm-be -1432914/AMC-TUT -1434670/8pen -1434732/3EBC -1434765/6thstreet1 -1434782/AllenHalsted -1434875/Andershouston -1437025/CFDEMproject -1436952/BrokenGravity -1434821/BPGRobot -1436086/CapCaval -1436358/Compassion -1434877/Cindygo -1435053/Dinamundo -1434800/Distie -1434809/Ekhrikhor -1436877/Engage -1436788/Exove -1436864/FinforeOrg -1436302/FoxyNinjaStudios -1436678/Flashtalking -1435470/G04 -1434933/FunSocketsGames -1436827/GlobalAndroidDevCamp -1435954/GizmoProjects -1434818/Gurke1993 -1435911/HGSSingen -1436015/HIC -1435276/InGamer -1435032/Jetpack-fr -1436745/KRMAssociatesInc -1435077/KartingCoach -1434797/Krishnab -1436050/LangExt -1434876/Kuraokami -1436035/MLSDev -1436848/Manacou -1434906/McMillinConstruction -1435598/MelianLabs -1434946/NextFaze -1434895/NTP -1435711/Nike-Inc -1435381/OOKB -1434912/OpenFlock -1435216/ParkingLotB -1434820/Pepper5 -1435141/ProjectC -1434796/PuNkYBoY -1434839/Qilver -1435559/RMS-TEAM -1435255/RSquaredSoftware -1434786/RandomFeatures -1434972/Rational-Interaction -1435083/Savings -1436844/SchillerConsulting -1435738/SoftwareDevelopmentCentre -1435731/StudentsDomainRegistry -1435511/Stunning -1436914/Symplectic -1434832/TSoP -1434792/The-Weather-Guy -1435794/TouchInstinct -1435510/VPAC -1436881/VassarEM -1436784/ZCA -1434785/ZacharySwanson -1434864/abclements -1434803/ajax333221 -1434805/alexandru-manea -1436814/alittlemarket -1436571/almende -1434857/andrearipamonti -1434837/applied-abstraction -1434831/arl1nd -1434817/asherswing -1434815/audiaganovic -1435647/axomi -1436406/axxxxs2011 -1434783/bcarey -1434801/benirwin -1434816/benquarmby -1434861/bfavrel -1436168/blogsport -1434824/bobbyobrien -1434874/bossal -1434885/cameronbrister -1436052/carawhite -1436905/cartomatic -1434794/ccakins -1434862/cec-david -1434851/christophercoy -1434788/chuotmusic -1436251/clever-lang -1435210/codemancode -1434848/cofercarv -1436170/colorfront -1436129/coub -1434855/cpalkar -1434847/craiggephart -1434880/cswoyer -1434887/cwang-sugar -1437096/cxdotxss -1434852/cyuseff -1435650/danbroadley -1434859/dandymich -1434872/daniel-wilson -1434835/danpollard -1436639/datadrivenjournalism -1435830/davinciez -1434807/dbakha -1434799/delta666 -1434798/derboehsevincent -1436120/domgeo -1434860/dovetail-hrdemo -1434802/drdelambre -1434810/efox2001 -1434791/ekstinehickory -1436044/eneongroup -1436657/epartment -1436557/essentialkaos -1435667/estilowebnet -1437040/gdgomsk -1434784/gerchenceto -1434825/hjschaeffer -1437063/hyenaDev -1436025/ideacreation -1437028/itcgames -1434833/jamesdemello -1434808/jeremysmith1 -1434849/jettenrules -1434884/joaquin666 -1436371/jolicode -1434830/jordan-chaput -1434793/jordandong -1436680/jugchennai -1434967/jvcreate -1434879/kamuflage661 -1434812/kenwyq -1434844/khmaies -1435313/kinishi -1434840/klape -1434819/kparthiban -1436630/lounaspaikka -1434846/lexmosolov -1434813/luism20 -1435005/mapaslivres -1434869/marihuertas -1435184/mcedit -1434853/mcpelletier -1434866/miguelisucorp -1434842/misteroo -1434827/mint7 -1434789/moneyconcepts -1434934/moq -1434814/moreone -1435972/mpdmal -1434881/mrki -1434806/mxtbcca -1436790/nanoblocks -1437068/nerdhive -1434878/ninu-deploy -1434858/njenia -1434787/nnicolasHippo -1434850/oliviernocent -1435197/otmsol -1434871/ozakaulov -1435463/packfire-archived -1434834/pandizajner -1434873/phollome -1434856/pixable-deploy -1434868/ppourmand -1437092/praisefm -1434838/prostohz -1436051/pyngo -1434804/raymond-isaac -1434795/realeuo -1435349/religionnews -1434822/rguedes -1434826/rpolasek -1434886/sega1601 -1437074/senseworldwide -1434998/shao-zhu7069 -1434829/shelakel -1434854/sourcescape -1434870/ssingh10 -1434823/strausz -1435823/sypProject -1434863/tatu-at-salesforce -1436910/teamgummy -1436329/testing12 -1434865/thehen -1435515/theorynetwork -1435668/thinkitglobal -1435439/typesafetest -1434845/ukrorion -1436990/uspublicsafety -1434843/wrecck -1435601/wwvx13 -1434882/xelat -1436648/ytstenut -1438047/100Plus -1438087/0x0101x0 -1439977/3EdgeTechnologies -1440029/282708579 -1441096/31337-team -1441423/ASU-CompSci -1441800/AWitness -1441804/7gano2 -1443938/AITHO -1443952/360ed-sapper -1445628/7Geese -1445634/0708jun -1445667/4T-Shirt -1446147/ADPSystems -1448136/350dotorg -1449096/AOS2012 -1451686/2Exist -1451715/0xPhilMill -1451721/0xbgr -1451828/8slater -1451928/0ibrahim -1452049/AndersonWillDo -1454981/ArcLight-Game-Studios -1457167/A24Group -1457261/4devel -1457403/ASNS -1458658/AXIS2012 -1458678/23kstudios -1461346/42qu -1463606/1800postcards -1463677/4Liamk -1463705/AfraidOfScaryBats -1466264/AmazingWorks -1468534/ASFINAG -1468552/3thanZ -1472258/Accounting-123 -1472385/1623dude -1472511/4194projext -1472584/209104768 -1474271/AU-COMP6360-2012Spring-Team2 -1474318/Andorion -1474413/Appwerks -1474498/1679tjapril -1474629/2hertz -1478478/AblosMedia -1480113/360PSG -1480194/112358D -1482134/17flying -1482248/11399371 -1483689/256Design -1483857/42069 -1485491/4commerce-technologies-AG -1486026/ANCL -1487607/ANSTO -1487679/Agerran -1487778/Allins -1490220/AnalyticalGraphicsInc -1490248/2cbarata -1493200/3Streams -1493210/123Aa -1496977/Acrowit -1497074/040107 -1497467/0bytesoftware -1500284/2dgreePublic -1501953/1701Dev -1504456/21tag -1507432/4SitePress -1507526/2011413125 -1507528/AllenYao -1507607/Alxgor -1507712/87borja -1507775/AdrianNabereit -1507800/AlexLovP -1510013/AugustFourth -1511395/54North -1512786/4Apps -1512024/AGILAD -1512907/ASYLUMSERVE-COMMUNICATIONS -1510104/Aaronaught -1512378/AmericanMediaInc -1510080/Aliaman76 -1512072/AntennaInternational -1510074/AtifChaudhry -1512594/BEST-FLAN-EVER -1511166/BelectricITS -1511960/Buckeyecocoa -1511554/Calpano -1510057/Chubbylilbuddy -1511031/Cinode-Labs -1512870/CirrusMio -1510601/CodeSick -1512668/CodeSimples -1510053/Comunidadphp -1511282/Cuprium -1512810/DanDotOrg -1510943/E-GENIUS -1512189/GVDistributed -1510073/E-Yoshimoto -1513056/ITOne -1510109/Geistyp -1512715/Invariance -1510036/KrithikaV -1510941/LesPoneys -1510032/LesignButure -1510634/LimeSoda -1510116/McTingler -1513176/Melati -1512881/Microsoft-Web -1510107/MichaelMunchin -1511261/MySettingsDevs -1510624/NarrativeTeam -1511360/NexwayGroup -1510049/OliviaXu -1512984/ProjektAugustiner -1510090/SpecterB -1510088/RMelnikas -1511178/RidiculousPower -1511948/Supintel2012 -1512676/TeleSign -1510069/Thor127 -1510992/Toppsolutions -1512096/Tripitch-Mediacreatives -1510111/WTFUSAY909 -1513271/Videodock -1510640/Workgroup -1511025/Xactium -1510775/YAYA-CSII2 -1512420/actmobile -1510054/ahashmi -1510072/alan1982br -1510071/alauria -1510038/aleinegreen -1510106/alexendersuperbyte -1510329/allwinnertech -1513245/ameronix -1510094/apacker -1510075/art-khokhlov -1510027/belleamme -1510083/benwhit -1513050/beta-nu-theta-chi -1510081/bladzaker -1510985/bluejayprojects -1512248/bluesubmarine -1512268/bmstu-rk6 -1510034/bodhiya -1510832/bonboa -1510055/bousan23 -1511467/brnkly -1512417/chrprtest -1512323/cloudarena -1510021/clzzclzz -1510618/colama -1510033/conkey33 -1510102/cosmogreco -1510030/crwbot -1512195/cryptosoftware -1510042/ctdeux -1510594/culturejam-cookbooks -1510028/damonsmith -1510061/darkDemon78 -1510084/daward13 -1510018/dhana12 -1510749/dignio -1510040/dijacobs -1510103/discoverbest808 -1513029/disekt -1510051/dritterman -1510025/dwi001 -1510029/eddieschneider -1510576/elfeon -1510110/ethan-th -1510469/euruko -1510026/fanxueliang31 -1510068/farafaeda -1512917/fesl -1511337/ffpojo -1512776/fligoo -1510098/francisbugaoan -1510039/gabsoftware -1510097/gdiocarez -1510537/hacolive -1510037/hash3 -1510019/haventownsend -1510062/hmtan1 -1513097/holonym -1510101/huichoperez -1512462/hydra-billing -1510043/ian-mac -1511267/informastudio -1510378/iseatz-ibs -1513210/iworx-greece -1510059/jayemko -1510114/jcowles -1510077/jenpan -1510020/jji7skyline -1510041/jmechler -1510022/jonto24 -1510047/jrtracing -1510113/jrupert -1510067/karnick -1510159/karrainc -1512985/lansing-makers-network -1510091/katafr -1511838/linux-test-project -1510087/leemh5544 -1510115/lyj289 -1512630/linuxmuster -1510883/millenoki -1510060/mkelleyjr -1510539/momoim -1511160/mozillahispano -1510078/mpeutz -1510045/muneebpv -1510017/nadeemkhan -1510014/nareg -1510515/netsolutionsindia -1512026/nizam5948 -1510066/notKBF -1510089/nynyn -1512771/origocms -1510493/ourTrue -1513343/paperkite -1510957/pixeltricks -1511325/playgen -1512031/pragmapool -1513215/psget -1513376/psu-oit-cis-unixteam -1510112/pwm1234 -1511339/railsarena -1510070/ranahammad -1510082/rhalas -1510052/rodrow -1510095/s8ndyleung -1510024/seemasjustin -1512944/simplicityio -1510085/sinergy -1510093/skepfish -1510064/smbcomputers -1510035/smbocmputers -1512430/splendia -1510092/suebatchelor -1510031/summersoulmate -1512971/sunnytechnologies -1512368/swiftmobile -1510044/sylvon -1510079/taitertot89 -1510046/tanhq -1510100/tavoav -1510416/terrestris -1510058/themaker -1512081/threemonkeyslab -1512631/tripode -1510076/ubidave -1510625/ultragalactic -1510096/uofr -1510105/uregina -1510108/van-hai-ho -1510597/verticallife -1512782/voidlinux -1510065/vrushank-snippets -1510086/vrushankpb -1510050/wenjian -1510016/whoever81 -1510015/willian12345 -1511686/xiaomen -1512387/zaizi -1510063/zanchu -1510099/zhfally520 -1514874/AMDISX -1514943/2dotsInc -1515025/AAXIS -1516139/AkademiskaSangforeningen -1516185/4apaev -1516249/B2melo -1516350/12pixels -1519519/Activateme -1521176/1stdibs -1523105/A639 -1523183/AndroidGadgetCyprus -1523230/0o0 -1523277/Alfred-Cai -1523351/Ade456 -1523445/AXMAN43 -1523535/2112bytes -1523558/AdminUIUX -1523662/AndroidGirl89 -1523706/123zar -1523738/619ist -1524834/BaliAutomation -1524922/23-Crandall -1527502/ArtworkCloud -1530272/1stEdge -1532307/14islands -1535312/16colossi -1535928/12things -1535970/15902977457 -1536057/44gi -1537361/907pine -1537639/ActionSprout -1536606/AkzoNobel -1536161/AlexMunro -1537813/Armada-Azteca -1538511/AskUbuntu -1538986/BTP400 -1536092/BorisFR -1537474/CS-IP -1537542/CS345 -1537101/CirclesPeople -1536830/Cierant -1536125/CroixANI -1536290/Deckmatt -1536115/DimaN-RuSlan -1537539/FundingOptions -1537696/Fleming-College -1537625/Gensler -1537617/GuimieSoft -1536768/HackCoMO -1536100/HanzGeeratz -1536856/IFAEDI -1536893/InVisionApp -1537177/IngloriousCoders -1539301/InterOffice -1536071/Isengua -1536876/JLGBHQ -1538614/Joinspeaker -1536363/Kinesis-io -1536165/LilyT -1537286/MSM7x27-Xperia -1536107/Manishhemnani -1537951/Militarizm -1536064/Nemanja100 -1538769/NextClimate -1537496/NexusResearchGroup -1536734/NorwegianBusinessSchool -1536973/RackTables -1538350/RogueGenius -1536067/SOGI-Ubuntu -1537685/SaintRock -1536720/ScienceMag -1536068/Scottimoto -1539762/ShareMyWorld -1537728/SilexConsulting -1536293/Skatteetaten -1536157/Smilev -1539068/SunEdison -1536139/StSch -1539332/TDEW-SAK -1536176/Telefonica -1538236/Toys-Games -1536795/ULCAMP -1536156/Un0 -1536862/Vluxe -1538535/WordPress-Phoenix -1536076/agent1987 -1536160/aglioeolio -1536066/ajstiefermann -1537757/amsrb -1536101/apandays -1537097/assettv -1536091/ashagi -1536123/ayaniv -1536078/bakkmyr -1536060/baturdem -1536089/be2z -1536088/bhupendra27 -1536069/bigthorzj -1536110/boozter -1536074/borisostojic -1538431/bristle-com -1538375/britishlibrary -1536140/candebatThibault -1536083/cdehner0 -1536135/cedef -1536079/cesards -1536116/chilfiger -1536058/chris-hmmr -1538725/clachic -1538395/cloudjams -1536179/code0 -1536825/combell -1536164/cosminonnet -1539603/cubbihouse -1538837/dailymuse -1536130/dannion -1538424/datazze -1538495/dayton-dynamic -1536163/ddsmes -1536081/debonatis -1536075/dhruvbhatz -1539555/dianping -1536073/diecastheroes -1538894/digitalworkbox -1536142/directionalpad -1536113/djengo-git -1536126/domage -1536978/domusvi -1537346/ebillett -1537651/edu4 -1536159/enanvad -1536062/endstern -1536097/eonyoung -1536158/epplerc -1536162/estingala -1539306/firadio -1538415/fulfillo -1536090/gagsy -1536065/gerhard-tinned -1536119/ghummer -1537917/greeenstarhub -1536059/guoyuanzhen321 -1536132/gvogets -1536087/hansso90 -1536103/hasanajsh -1536105/hpetersen -1538309/hsbp -1536155/jaeho-kim -1536080/josman17 -1536086/jpick -1536106/kheatsopon -1537518/kickinespresso -1536141/kimhist -1536098/kjersva -1539155/knovce -1536111/konri1990 -1538478/koradomi -1536148/krishnabanala -1536393/londonwebstandards -1536122/mankeyl -1536117/massproduction -1536151/mconrads -1536127/michaelzhang2012 -1538694/mindpin-shcmusic -1536138/minzirillo -1536096/misha-at-genestack -1536131/mklunker -1537561/mobiManagellc -1536144/mosibasad -1538317/netmarkllc -1536523/ngs-doo -1536063/oinkgms -1536826/okcupidlabs -1537857/okcupidlabz -1536835/opensoftwareconsulting -1536084/owenllharris -1537894/partnersandnapier -1538133/perceus -1539576/pfe2012 -1536121/piyalib -1536184/presemt-ntnu -1537562/rdmapps -1537935/redtruck -1536367/rentify -1539476/reseau-loops -1537354/rgbhs -1536153/samweston -1536136/sarita-veeramalla -1536085/sayuriyajima -1536149/sealand210 -1536070/sheeta -1536145/sgatev -1536099/shinisuba -1536114/sindarfin -1536120/sodhibhupinder -1536124/souflyte -1536112/sp-javi-garcia -1536147/sryoung06 -1536108/ssswang -1536077/streamingcrusher -1537811/talintelIT -1536128/tapthat25 -1539715/tdt4258-somagisk -1538081/tdw-grupo-yfb -1536134/teemuroos -1536640/telldus -1537872/tool2team -1538058/tradenetservicessrl -1538676/tuftandneedle -1537483/twakooltd -1538578/ubhacking -1536093/udzhou -1536152/vasdev04 -1536143/venki58 -1536061/vfuente -1536133/void-phoenix -1536102/wakusei-meron- -1537342/webforall -1536146/wisaly -1539627/whitesource -1536095/wuminghello -1536521/yodle -1537695/younglives -1536129/zengooo -1536150/zpardos diff --git a/src/lib/owner.sh b/src/lib/owner.sh index a4fcdf02e9b21..4a3fa56b87ea2 100644 --- a/src/lib/owner.sh +++ b/src/lib/owner.sh @@ -124,12 +124,14 @@ update_owner() { find "$BKG_INDEX_DIR/$owner" -type f -name '*.json' ! -name '.*' -print0 | xargs -0 jq -cs '[.] | add' >"$BKG_INDEX_DIR/$owner/.json.tmp" jq -cs '{ ("package"): . }' "$BKG_INDEX_DIR/$owner/.json.tmp" >"$BKG_INDEX_DIR/$owner/.json" ytoxt "$BKG_INDEX_DIR/$owner/.json" + jq -c '.package[]' "$BKG_INDEX_DIR/$owner/.json" >"$BKG_INDEX_DIR/$owner/.json.tmp" mv -f "$BKG_INDEX_DIR/$owner/.json.tmp" "$BKG_INDEX_DIR/$owner/.json" echo "Creating $owner repo arrays..." parallel "jq -c --arg repo {} '[.[] | select(.repo == \$repo)]' \"$BKG_INDEX_DIR/$owner/.json\" > \"$BKG_INDEX_DIR/$owner/{}/.json.tmp\"" <<<"$owner_repos" xargs -I {} bash -c "jq -cs '{ (\"package\"): . }' \"$BKG_INDEX_DIR/$owner/{}/.json.tmp\" > \"$BKG_INDEX_DIR/$owner/{}/.json\"" <<<"$owner_repos" xargs -I {} bash -c "ytoxt \"$BKG_INDEX_DIR/$owner/{}/.json\"" <<<"$owner_repos" + xargs -I {} bash -c "jq -c '.package[]' \"$BKG_INDEX_DIR/$owner/{}/.json\" > \"$BKG_INDEX_DIR/$owner/{}/.json.tmp\"" <<<"$owner_repos" xargs -I {} mv -f "$BKG_INDEX_DIR/$owner/{}/.json.tmp" "$BKG_INDEX_DIR/$owner/{}/.json" <<<"$owner_repos" fi diff --git a/src/lib/util.sh b/src/lib/util.sh index ae24ce69d29dd..d4c8569af6f88 100644 --- a/src/lib/util.sh +++ b/src/lib/util.sh @@ -401,6 +401,8 @@ curl_orgs() { explore() { local node=$1 + local is_user=false + local got_orgs=false [[ "$node" =~ .*\/.* ]] && local graph=("stargazers" "watchers" "forks") || local graph=("followers" "following" "people") [ -z "$2" ] || graph=("$2") @@ -412,12 +414,19 @@ explore() { if [[ "$node" =~ .*\/.* ]]; then nodes=$(curl_users "$node/$edge?page=$page") # repo else - nodes=$(curl_users "orgs/$node/$edge?page=$page") # org - - if [ -z "$nodes" ]; then - nodes=$(curl_users "$node?tab=$edge&page=$page") # user - curl_orgs "$1" - fi + if [ "$is_user" = false ]; then + nodes=$(curl_users "orgs/$node/$edge?page=$page") # org + [ -n "$nodes" ] || is_user=true + fi + + if [ "$is_user" = true ]; then + nodes=$(curl_users "$node?tab=$edge&page=$page") # user + + if [ "$got_orgs" = false ]; then + curl_orgs "$node" + got_orgs=true + fi + fi fi grep -v "$(cut -d'/' -f1 <<<"$node")" <<<"$nodes" @@ -444,62 +453,200 @@ ytox() { } ytoxt() { - # ytox + trim if the json or xml is over 50MB, remove oldest versions - while [ -f "$1" ] && [[ "$(ytox "$1")" -ge 50000000 || "$(stat -c %s "$1")" -ge 50000000 ]]; do + # ytox + trim: if the json or xml is over 50MB, remove oldest versions + local f="$1" + local tmp + local del_n=1 + local last_xml_size=-1 + + [ -f "$f" ] || return 1 + + tmp=$(mktemp "${f}.XXXXXX") || return 1 + trap 'rm -f "$tmp"' RETURN + + while [ -f "$f" ]; do + local json_size + local xml_size + local tmp_size + + json_size=$(stat -c %s "$f" 2>/dev/null || echo -1) + + if [ "$json_size" -lt 50000000 ]; then + # Only generate/check XML if JSON is already under limit. + xml_size=$(ytox "$f" 2>/dev/null || echo -1) + # If XML size can't be determined, treat it as oversized so we keep trimming. + [ "$xml_size" -ge 0 ] || xml_size=50000000 + + if [ "$xml_size" -lt 50000000 ]; then + break + fi + + # If XML is still too large, keep trimming, but avoid redoing work forever. + if [ "$xml_size" -eq "$last_xml_size" ] && [ "$last_xml_size" -ge 0 ]; then + break + fi + last_xml_size="$xml_size" + + # XML still too large: increase trimming aggressiveness as well. + if [ "$del_n" -lt 65536 ]; then + del_n=$((del_n * 2)) + fi + else + # JSON is still too large: increase trimming aggressiveness. + if [ "$json_size" -ge 50000000 ]; then + if [ "$del_n" -lt 65536 ]; then + del_n=$((del_n * 2)) + fi + fi + fi + if jq -e ' - if type == "array" then - any(.[]; (.version // []) | length > 0) - else - (.version // []) | length > 0 - end - ' "$1" >/dev/null; then + if (type == "array") or (type == "object") then + any(.[]; ((.version // []) | type == "array") and ((.version // []) | length > 0)) + else + ((.version // []) | type == "array") and ((.version // []) | length > 0) + end + ' "$f" >/dev/null; then + jq -c ' + def id_to_num: + if type == "number" then . + elif type == "string" then tonumber? // 0 + else 0 end; + def vlen: + (.version // []) | if type == "array" then length else 0 end; + def trim_versions($n): + if ((.version // []) | type == "array") and ((.version // []) | length > 0) then + ( + .version + | sort_by(.id | id_to_num) + | .[$n:] + ) as $v + | .version = $v + else + . + end; + if type == "array" then + (to_entries + | (max_by(.value | vlen) // empty) as $max + | map( + if .key == $max.key and ((.value | vlen) > 0) + then (.value |= trim_versions($n)) + else . + end + ) + | map(.value)) + elif type == "object" then + (to_entries + | (max_by(.value | vlen) // empty) as $max + | map( + if .key == $max.key and ((.value | vlen) > 0) + then (.value |= trim_versions($n)) + else . + end + ) + | from_entries) + else + trim_versions($n) + end + ' --argjson n "$del_n" "$f" >"$tmp" + else jq -c ' - def trim_versions: - if (.version // []) | length > 0 then - .version |= ( - sort_by(.id | (if type == "string" and test("^-?[0-9]+$") then tonumber else . end)) - | del(.[0]) - ) - else - . - end; - if type == "array" then - (to_entries - | (max_by((.value.version // []) | length) // empty) as $max - | map( - if .key == $max.key and ((.value.version // []) | length > 0) - then (.value |= trim_versions) - else . - end - ) - | map(.value)) - else - trim_versions - end - ' "$1" >"$1".tmp - else - jq -c ' - if type == "array" then - ( - def to_num: - if type == "number" then . - elif type == "string" then tonumber? // 0 - else 0 end; - to_entries - | (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target - | if $target == null then - map(.value) - else - [ .[] | select(.key != $target.key) | .value ] - end - ) - else - . - end - ' "$1" >"$1".tmp + if type == "array" then + ( + def to_num: + if type == "number" then . + elif type == "string" then tonumber? // 0 + else 0 end; + to_entries + | (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target + | if $target == null then + map(.value) + else + [ .[] | select(.key != $target.key) | .value ] + end + ) + elif type == "object" then + ( + def to_num: + if type == "number" then . + elif type == "string" then tonumber? // 0 + else 0 end; + to_entries + | (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target + | if $target == null then + from_entries + else + ([ .[] | select(.key != $target.key) ] | from_entries) + end + ) + else + . + end + ' "$f" >"$tmp" fi - mv "$1".tmp "$1" + + tmp_size=$(stat -c %s "$tmp" 2>/dev/null || echo -1) + + # If trimming didn't reduce size, retry with more aggressive deletion instead of stalling. + if [ "$json_size" -ge 0 ] && [ "$tmp_size" -ge 0 ] && [ "$tmp_size" -ge "$json_size" ]; then + rm -f "$tmp" + + if [ "$del_n" -lt 65536 ]; then + del_n=$((del_n * 2)) + continue + fi + + # If we're already at max aggressiveness, fall back to trimming whole packages once. + jq -c ' + if type == "array" then + ( + def to_num: + if type == "number" then . + elif type == "string" then tonumber? // 0 + else 0 end; + to_entries + | (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target + | if $target == null then + map(.value) + else + [ .[] | select(.key != $target.key) | .value ] + end + ) + elif type == "object" then + ( + def to_num: + if type == "number" then . + elif type == "string" then tonumber? // 0 + else 0 end; + to_entries + | (min_by([ (.value.raw_downloads // 0 | to_num), (.value.date // "") ]) // null) as $target + | if $target == null then + from_entries + else + ([ .[] | select(.key != $target.key) ] | from_entries) + end + ) + else + . + end + ' "$f" >"$tmp" + + tmp_size=$(stat -c %s "$tmp" 2>/dev/null || echo -1) + if [ "$json_size" -ge 0 ] && [ "$tmp_size" -ge 0 ] && [ "$tmp_size" -ge "$json_size" ]; then + rm -f "$tmp" + break + fi + fi + + mv "$tmp" "$f" done + + # Ensure the XML output corresponds to the final JSON. + ytox "$f" >/dev/null 2>&1 + + # If either JSON or XML is > 100MB, empty each one that is too large: + [ "$(stat -c %s "$f" 2>/dev/null || echo -1)" -lt 100000000 ] || echo "{}" >"$f" + [ "$(stat -c %s "${f%.*}.xml" 2>/dev/null || echo -1)" -lt 100000000 ] || echo "" >"${f%.*}.xml" } ytoy() {