1

http://debank.com/ranking/locked_value This is the website that I am looking to scrape and get all the tokens out with the rankings in another column.

below is the output I have gotten to read the url above in html. However, the way the main body is printed out below looks different than inspecting the element directly on the website. Is it because of the noscript tag? I would also like to know if its even possible to scrape this website with BeautifulSoup first off by looking at the html on the website for it and the way the data is structured? I have tried importhtml and importxml on GoogleSheets but it doesn't work as well.

 <body>
  <noscript>
   You need to enable JavaScript to run this app.
  </noscript>
  <div id="root">
  </div>
  <script>
   !function(i){function e(e){for(var t,r,n=e[0],a=e[1],o=e[2],c=0,f=[];c<n.length;c++)r=n[c],Object.prototype.hasOwnProperty.call(s,r)&&s[r]&&f.push(s[r][0]),s[r]=0;for(t in a)Object.prototype.hasOwnProperty.call(a,t)&&(i[t]=a[t]);for(h&&h(e);f.length;)f.shift()();return d.push.apply(d,o||[]),u()}function u(){for(var e,t=0;t<d.length;t++){for(var r=d[t],n=!0,a=1;a<r.length;a++){var o=r[a];0!==s[o]&&(n=!1)}n&&(d.splice(t--,1),e=p(p.s=r[0]))}return e}var r={},l={10:0},s={10:0},d=[];function p(e){if(r[e])return r[e].exports;var t=r[e]={i:e,l:!1,exports:{}};return i[e].call(t.exports,t,t.exports,p),t.l=!0,t.exports}p.e=function(d){var e=[];l[d]?e.push(l[d]):0!==l[d]&&{2:1,12:1,13:1,14:1,15:1,16:1,17:1,18:1,19:1,20:1,22:1,23:1}[d]&&e.push(l[d]=new Promise(function(e,n){for(var t="static/css/"+({}[d]||d)+"."+{2:"31a01016",12:"801c88c0",13:"e96e5493",14:"1bdcb274",15:"77827393",16:"ce78dcea",17:"1bf44b6e",18:"db6a371a",19:"a9b0ed7a",20:"a1c112e5",21:"31d6cfe0",22:"7af2c859",23:"a274fc5e",24:"31d6cfe0",25:"31d6cfe0",26:"31d6cfe0",27:"31d6cfe0",28:"31d6cfe0",29:"31d6cfe0",30:"31d6cfe0"}[d]+".chunk.css",a=p.p+t,r=document.getElementsByTagName("link"),o=0;o<r.length;o++){var c=(i=r[o]).getAttribute("data-href")||i.getAttribute("href");if("stylesheet"===i.rel&&(c===t||c===a))return e()}var f=document.getElementsByTagName("style");for(o=0;o<f.length;o++){var i;if((c=(i=f[o]).getAttribute("data-href"))===t||c===a)return e()}var u=document.createElement("link");u.rel="stylesheet",u.type="text/css",u.onload=e,u.onerror=function(e){var t=e&&e.target&&e.target.src||a,r=new Error("Loading CSS chunk "+d+" failed.\n("+t+")");r.code="CSS_CHUNK_LOAD_FAILED",r.request=t,delete l[d],u.parentNode.removeChild(u),n(r)},u.href=a,document.getElementsByTagName("head")[0].appendChild(u)}).then(function(){l[d]=0}));var r=s[d];if(0!==r)if(r)e.push(r[2]);else{var t=new Promise(function(e,t){r=s[d]=[e,t]});e.push(r[2]=t);var n,a=document.createElement("script");a.charset="utf-8",a.timeout=120,p.nc&&a.setAttribute("nonce",p.nc),a.src=p.p+"static/js/"+({}[d]||d)+"."+{2:"dfdf9b30",12:"f7c27ba6",13:"858ad66b",14:"8ca994df",15:"9b1dce21",16:"5cb6bdad",17:"093dc229",18:"49ea6a2c",19:"5d3d7d18",20:"a0f26441",21:"f638e05d",22:"408bed83",23:"7910c8db",24:"6c339fe5",25:"e3ae2970",26:"a0fc75ba",27:"62eeed18",28:"5c2ce104",29:"f5c8a1a0",30:"9bf66e26"}[d]+".chunk.js";var o=new Error;n=function(e){a.onerror=a.onload=null,clearTimeout(c);var t=s[d];if(0!==t){if(t){var r=e&&("load"===e.type?"missing":e.type),n=e&&e.target&&e.target.src;o.message="Loading chunk "+d+" failed.\n("+r+": "+n+")",o.name="ChunkLoadError",o.type=r,o.request=n,t[1](o)}s[d]=void 0}};var c=setTimeout(function(){n({type:"timeout",target:a})},12e4);a.onerror=a.onload=n,document.head.appendChild(a)}return Promise.all(e)},p.m=i,p.c=r,p.d=function(e,t,r){p.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:r})},p.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},p.t=function(t,e){if(1&e&&(t=p(t)),8&e)return t;if(4&e&&"object"==typeof t&&t&&t.__esModule)return t;var r=Object.create(null);if(p.r(r),Object.defineProperty(r,"default",{enumerable:!0,value:t}),2&e&&"string"!=typeof t)for(var n in t)p.d(r,n,function(e){return t[e]}.bind(null,n));return r},p.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return p.d(t,"a",t),t},p.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},p.p="/",p.oe=function(e){throw console.error(e),e};var t=this["webpackJsonpdefi-insight-react"]=this["webpackJsonpdefi-insight-react"]||[],n=t.push.bind(t);t.push=e,t=t.slice();for(var a=0;a<t.length;a++)e(t[a]);var h=n;u()}([])
jamie
  • 11
  • 2
  • Cloudflare is throwing a host error for that site at the moment. Apart from that, it seems that you're scraping a site with content dynamically loaded with `javascript`. Using `requests` won't capture dynamic content. Check [this question](https://stackoverflow.com/questions/8049520/web-scraping-javascript-page-with-python) for solutions. – RJ Adriaansen Apr 22 '21 at 08:37

1 Answers1

0

The data is loaded from external URL via Javascript. You can use this example how to load the data using requests/json:

import json
import requests

url = "https://static1.debank.com/tvl/projects.json"
data = requests.get(url).json()

# uncomment this to print all data:
# print(json.dumps(data, indent=4))

# print some data:
for chain in data:
    print(chain["name"]["en"])

Prints:

1inch
88mph V1
88mph V2
Aave V1
Aave V2
Aave AMM
acBTC
Alchemix
Alpaca City
AnySwap
Armor
Async Art
Balancer
Bancor
BarnBridge
Base
Basis Cash
Basis Coin
Basis Dollar
Basis Gold
Big Data Protocol
BoringDAO
B.Protocol
Alpaca Finance
BakerySwap
bDollar
BSCswap
Cream
dfuture
DODO
Ellipsis
Goose Finance
JulSwap
MDEX
Nerve
PancakeSwap
PureSwap
QIAN
Smoothy
Venus
bZx
ChainLink
Charm
ChickenSwap
CoFiX
component
Compound
Connext
CoreVault
COVER
Cream
Curve
Daiq
DDEX
Deerfi
DeFiDollar
DeFiner
DeFi Swap
Dego
DerivaDEX
DeversiFi
dForce Lending
dHEDGE
DefiMoneyMarket
dmm.exchange
DODO
Doki Doki
Dracula
DSD
dForce Yield Markets
DuckDao
dYdX
Empty Set Dollar
Equalizer
FairSwap
Fei Protocol
Ferrum
FinNexus
FIREDAO
Float
ForTube Bank
ForTube Bond
Frax
Freeliquid
Furucombo
Futureswap
Gnosis
Golff
GrowthDeFi
GYSR
HAKKA
Hegic
HyperBC Farm
ichi.farm
Idle
Index Coop
Indexed
Keep3r
KeeperDAO
KIMCHI
Kine
Kyber
Leverj
Liquity
Loopring V1
Loopring
LuaSwap
Maker
Benchmark(MARK)
Aave
ComethSwap
Dfyn
easyfi
QuickSwap
MCDEX
Meme
Mercurity
M.finance
MiniSwap
Mirror
MITH.CASH
Mooniswap
mStable
Mushrooms
Nest
Nexus Mutual
Notional
Nsure Network
Nuo
One Cash
OneSwap
OpenDAO
OpenSea
Opium
Opyn V1
Opyn
Orai
Orion Protocol
PEAKDEFI
Percent
PerlinX
Perpetual
PoolTogether
PowerPool
QIAN
Radix
RAMP DEFI
Rari Capital
Reflexer
Ren
Reserve
Rootkit
Sablier
Saddle
Saffron
SakeSwap
SashimiSwap
S.Finance
Shell
Siren
Smoothy
SnowSwap
Stabilize
Stand Cash
STEROIDS
STONE
Sunflower Swap
SushiSwap
Swerve
SWISS
SynLev
Synthetix
Tellor
Tokenlon
TokenSets
Tornado
TrueFi
TrustSwap
TyphoonCash
UMA
Unagii
Unimex
Unipower
Unisave
Uniswap V1
Uniswap
unit protocol
unn
Unslashed
Warp
WePiggy
xDai
Bao.Finance
component
Honeyswap
X SIGMA
XUSD
YAM V1
YfDAI
YFLink
YIELD (yield.is)
YVS
ZAI V1
ZKSwap
Andrej Kesely
  • 81,807
  • 10
  • 31
  • 56
  • hi, thanks for the help. How do I go about getting the tokens out in order of the website's ranking? Would want to use webscraping to get the data because the ranking constantly changes over time as well. – jamie Apr 23 '21 at 04:53