[{"data":1,"prerenderedAt":549},["ShallowReactive",2],{"project-ensemble":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"name":10,"featured":11,"year":12,"topics":13,"heroImage":17,"archived":6,"languages":20,"homepage":27,"githubUrl":28,"tools":29,"body":36,"_type":543,"_id":544,"_source":545,"_file":546,"_stem":547,"_extension":548},"/projects/ensemble","projects",false,"","Ensemble","Visualise musical connections ✨","ensemble",true,2023,[14,15,16],"graph analysis","web scraping","performance testing",{"remote":18,"local":19},"https://raw.githubusercontent.com/ryanachten/ensemble/main/docs/ensemble_splash.png","hero-images/ensemble.webp",[21,22,23,24,25,26],"Go","TypeScript","Svelte","JavaScript","HTML","CSS","https://ryanachten.github.io/ensemble/","https://github.com/ryanachten/ensemble",[30,31,32,33,34,35],"gin","github-actions","k6","nodejs","sveltekit","tailwindcss",{"type":37,"children":38,"toc":530},"root",[39,50,56,64,71,76,84,91,96,105,157,165,199,207,234,245,251,256,264,270,283,291,297,311,319,325,330,338,352,366,374,406,413,418,430,464,469,483,491,497],{"type":40,"tag":41,"props":42,"children":43},"element","p",{},[44],{"type":40,"tag":45,"props":46,"children":49},"img",{"alt":47,"src":48},"ensemble splash","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_splash.png",[],{"type":40,"tag":41,"props":51,"children":52},{},[53],{"type":54,"value":55},"text","Web application visualising connections between bands, artists and genres.",{"type":40,"tag":41,"props":57,"children":58},{},[59],{"type":40,"tag":45,"props":60,"children":63},{"alt":61,"src":62},"ensemble screenshot","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_screenshot_rollingstones.jpg",[],{"type":40,"tag":65,"props":66,"children":68},"h2",{"id":67},"overview",[69],{"type":54,"value":70},"Overview",{"type":40,"tag":41,"props":72,"children":73},{},[74],{"type":54,"value":75},"Ensemble uses Wikipedia as a data source by scraping metadata for a given query and recursively visiting relevant links. These relationships are dynamically built up as an adjacency list before being formatted for client consumption once a given degree-of-separation is met.",{"type":40,"tag":41,"props":77,"children":78},{},[79],{"type":40,"tag":45,"props":80,"children":83},{"alt":81,"src":82},"ensemble sequence","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_sequence.png",[],{"type":40,"tag":85,"props":86,"children":88},"h3",{"id":87},"stack",[89],{"type":54,"value":90},"Stack",{"type":40,"tag":41,"props":92,"children":93},{},[94],{"type":54,"value":95},"Ensemble is built using the following technologies:",{"type":40,"tag":41,"props":97,"children":98},{},[99],{"type":40,"tag":100,"props":101,"children":102},"strong",{},[103],{"type":54,"value":104},"Client",{"type":40,"tag":106,"props":107,"children":108},"ul",{},[109,124,145],{"type":40,"tag":110,"props":111,"children":112},"li",{},[113,122],{"type":40,"tag":114,"props":115,"children":119},"a",{"href":116,"rel":117},"https://svelte.dev/",[118],"nofollow",[120],{"type":54,"value":121},"Svelte + Sveltekit",{"type":54,"value":123}," - Client framework",{"type":40,"tag":110,"props":125,"children":126},{},[127,134,136,143],{"type":40,"tag":114,"props":128,"children":131},{"href":129,"rel":130},"https://tailwindcss.com/",[118],[132],{"type":54,"value":133},"Tailwind",{"type":54,"value":135}," + ",{"type":40,"tag":114,"props":137,"children":140},{"href":138,"rel":139},"https://daisyui.com/",[118],[141],{"type":54,"value":142},"DaisyUI",{"type":54,"value":144}," - Styling",{"type":40,"tag":110,"props":146,"children":147},{},[148,155],{"type":40,"tag":114,"props":149,"children":152},{"href":150,"rel":151},"https://js.cytoscape.org/",[118],[153],{"type":54,"value":154},"Cytoscape",{"type":54,"value":156}," - Graph rendering and searching",{"type":40,"tag":41,"props":158,"children":159},{},[160],{"type":40,"tag":100,"props":161,"children":162},{},[163],{"type":54,"value":164},"API",{"type":40,"tag":106,"props":166,"children":167},{},[168,187],{"type":40,"tag":110,"props":169,"children":170},{},[171,177,178,185],{"type":40,"tag":114,"props":172,"children":175},{"href":173,"rel":174},"https://go.dev/",[118],[176],{"type":54,"value":21},{"type":54,"value":135},{"type":40,"tag":114,"props":179,"children":182},{"href":180,"rel":181},"https://gin-gonic.com/",[118],[183],{"type":54,"value":184},"Gin",{"type":54,"value":186}," - API framework",{"type":40,"tag":110,"props":188,"children":189},{},[190,197],{"type":40,"tag":114,"props":191,"children":194},{"href":192,"rel":193},"https://go-colly.org/",[118],[195],{"type":54,"value":196},"Colly",{"type":54,"value":198}," - Web scraping",{"type":40,"tag":41,"props":200,"children":201},{},[202],{"type":40,"tag":100,"props":203,"children":204},{},[205],{"type":54,"value":206},"Performance testing",{"type":40,"tag":106,"props":208,"children":209},{},[210,222],{"type":40,"tag":110,"props":211,"children":212},{},[213,220],{"type":40,"tag":114,"props":214,"children":217},{"href":215,"rel":216},"https://nodejs.org/en",[118],[218],{"type":54,"value":219},"Node",{"type":54,"value":221}," - tooling",{"type":40,"tag":110,"props":223,"children":224},{},[225,232],{"type":40,"tag":114,"props":226,"children":229},{"href":227,"rel":228},"https://k6.io/",[118],[230],{"type":54,"value":231},"K6",{"type":54,"value":233}," - performance testing",{"type":40,"tag":41,"props":235,"children":236},{},[237,241],{"type":40,"tag":45,"props":238,"children":240},{"alt":61,"src":239},"https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_screenshot_blackflag.jpg",[],{"type":40,"tag":45,"props":242,"children":244},{"alt":61,"src":243},"https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_screenshot_royorbison.jpg",[],{"type":40,"tag":65,"props":246,"children":248},{"id":247},"graph-relationships",[249],{"type":54,"value":250},"Graph relationships",{"type":40,"tag":41,"props":252,"children":253},{},[254],{"type":54,"value":255},"In the band or artist graph, the following relationships are supported in addition to their corresponding data source in the Wikipedia website. The genre graph is a little different, where only genre nodes are currently included.",{"type":40,"tag":41,"props":257,"children":258},{},[259],{"type":40,"tag":45,"props":260,"children":263},{"alt":261,"src":262},"ensemble band nodes","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_band_nodes.png",[],{"type":40,"tag":65,"props":265,"children":267},{"id":266},"degrees-of-separation",[268],{"type":54,"value":269},"Degrees of Separation",{"type":40,"tag":41,"props":271,"children":272},{},[273,275,281],{"type":54,"value":274},"To prevent infinite recursion, we need a hard limit how how many times to recursion cycles we'll complete when building the graph. In the Ensemble, this is limit is referred to as ",{"type":40,"tag":276,"props":277,"children":278},"em",{},[279],{"type":54,"value":280},"degrees of separation",{"type":54,"value":282}," from the original target.",{"type":40,"tag":41,"props":284,"children":285},{},[286],{"type":40,"tag":45,"props":287,"children":290},{"alt":288,"src":289},"ensemble degrees of separation","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_dos.png",[],{"type":40,"tag":65,"props":292,"children":294},{"id":293},"pathfinding",[295],{"type":54,"value":296},"Pathfinding",{"type":40,"tag":41,"props":298,"children":299},{},[300,302,309],{"type":54,"value":301},"Currently, Ensemble uses Cytoscape's implementation of the ",{"type":40,"tag":114,"props":303,"children":306},{"href":304,"rel":305},"https://js.cytoscape.org/#eles.floydWarshall",[118],[307],{"type":54,"value":308},"Floyd-Warshall",{"type":54,"value":310}," search algorithm for finding the shortest paths between two nodes. Genre nodes are weighted slightly lower than band or artist nodes to discourage routing via genres for more interesting paths.",{"type":40,"tag":41,"props":312,"children":313},{},[314],{"type":40,"tag":45,"props":315,"children":318},{"alt":316,"src":317},"ensemble pathfinding","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_screenshot_wutang.jpg",[],{"type":40,"tag":65,"props":320,"children":322},{"id":321},"concurrency",[323],{"type":54,"value":324},"Concurrency",{"type":40,"tag":41,"props":326,"children":327},{},[328],{"type":54,"value":329},"The initial approach when scraping information from Wikipedia and building up the graph was by sequentially visiting each node using depth-first search.\nThis approach produced a bottleneck awaiting HTTP requests to Wikipedia to retrieve node metadata.",{"type":40,"tag":41,"props":331,"children":332},{},[333],{"type":40,"tag":45,"props":334,"children":337},{"alt":335,"src":336},"ensemble strategies","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_strategies.png",[],{"type":40,"tag":41,"props":339,"children":340},{},[341,343,350],{"type":54,"value":342},"To address this HTTP request bottleneck, node visitations were parallelized using ",{"type":40,"tag":114,"props":344,"children":347},{"href":345,"rel":346},"https://go.dev/tour/concurrency/1",[118],[348],{"type":54,"value":349},"goroutines",{"type":54,"value":351},". This produced a secondary issue, where concurrent read and writes lead to data loss when compared to the graph produced sequentially.",{"type":40,"tag":41,"props":353,"children":354},{},[355,357,364],{"type":54,"value":356},"To address concurrent read and write conflicts, graph updates were sequenced using a queue. A Go ",{"type":40,"tag":114,"props":358,"children":361},{"href":359,"rel":360},"https://go.dev/tour/concurrency/2",[118],[362],{"type":54,"value":363},"channel",{"type":54,"value":365}," watches for queue entries and executes graph updates accordingly.",{"type":40,"tag":41,"props":367,"children":368},{},[369],{"type":40,"tag":45,"props":370,"children":373},{"alt":371,"src":372},"ensemble graphs","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_graph_models.png",[],{"type":40,"tag":41,"props":375,"children":376},{},[377,379,386,388,395,397,404],{"type":54,"value":378},"There are a couple of ways we can create thread-safe graphs in Golang; the ",{"type":40,"tag":380,"props":381,"children":383},"code",{"className":382},[],[384],{"type":54,"value":385},"sync",{"type":54,"value":387}," package comes with a concurrent-safe ",{"type":40,"tag":114,"props":389,"children":392},{"href":390,"rel":391},"https://pkg.go.dev/sync#Map",[118],[393],{"type":54,"value":394},"Map",{"type":54,"value":396},", or we can implement our own synchronisation using ",{"type":40,"tag":114,"props":398,"children":401},{"href":399,"rel":400},"https://pkg.go.dev/sync#Mutex",[118],[402],{"type":54,"value":403},"Mutex",{"type":54,"value":405}," mutual exclusion locks. We decided to test both approaches to see which would yield the best results when compared to the sequential approach.",{"type":40,"tag":41,"props":407,"children":408},{},[409],{"type":40,"tag":45,"props":410,"children":412},{"alt":316,"src":411},"https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_screenshot_sleep.jpg",[],{"type":40,"tag":65,"props":414,"children":416},{"id":415},"performance-testing",[417],{"type":54,"value":206},{"type":40,"tag":41,"props":419,"children":420},{},[421,423,428],{"type":54,"value":422},"To evaluate which strategy and graph implementation performed the best, we created performance tests using ",{"type":40,"tag":114,"props":424,"children":426},{"href":227,"rel":425},[118],[427],{"type":54,"value":231},{"type":54,"value":429},". We tested for 3 different factors:",{"type":40,"tag":431,"props":432,"children":433},"ol",{},[434,444,454],{"type":40,"tag":110,"props":435,"children":436},{},[437,442],{"type":40,"tag":100,"props":438,"children":439},{},[440],{"type":54,"value":441},"Reliability",{"type":54,"value":443}," - did data loss occur",{"type":40,"tag":110,"props":445,"children":446},{},[447,452],{"type":40,"tag":100,"props":448,"children":449},{},[450],{"type":54,"value":451},"Latency",{"type":54,"value":453}," - how long did the graph take to generate",{"type":40,"tag":110,"props":455,"children":456},{},[457,462],{"type":40,"tag":100,"props":458,"children":459},{},[460],{"type":54,"value":461},"Request failure",{"type":54,"value":463}," - did the response return a non-200 status code",{"type":40,"tag":41,"props":465,"children":466},{},[467],{"type":54,"value":468},"These were also evaluated using different numbers of concurrent users and degrees of separation.",{"type":40,"tag":41,"props":470,"children":471},{},[472,474,481],{"type":54,"value":473},"A custom test runner was built in Node to execute these tests and format the results in CSV for client display - the charted results can be viewed on the ",{"type":40,"tag":114,"props":475,"children":478},{"href":476,"rel":477},"https://ryanachten.github.io/ensemble/stats",[118],[479],{"type":54,"value":480},"Ensemble website",{"type":54,"value":482},".",{"type":40,"tag":41,"props":484,"children":485},{},[486],{"type":40,"tag":45,"props":487,"children":490},{"alt":488,"src":489},"ensemble performance testing","https://github.com/ryanachten/ensemble/raw/main/docs/ensemble_performance.png",[],{"type":40,"tag":65,"props":492,"children":494},{"id":493},"further-information",[495],{"type":54,"value":496},"Further information",{"type":40,"tag":106,"props":498,"children":499},{},[500,510,520],{"type":40,"tag":110,"props":501,"children":502},{},[503],{"type":40,"tag":114,"props":504,"children":507},{"href":505,"rel":506},"https://github.com/ryanachten/ensemble/raw/main/client/README.md",[118],[508],{"type":54,"value":509},"Running ensemble client",{"type":40,"tag":110,"props":511,"children":512},{},[513],{"type":40,"tag":114,"props":514,"children":517},{"href":515,"rel":516},"https://github.com/ryanachten/ensemble/raw/main/api/README.md",[118],[518],{"type":54,"value":519},"Running ensemble API",{"type":40,"tag":110,"props":521,"children":522},{},[523],{"type":40,"tag":114,"props":524,"children":527},{"href":525,"rel":526},"https://github.com/ryanachten/ensemble/raw/main/performance/README.md",[118],[528],{"type":54,"value":529},"Running ensemble performance tests",{"title":7,"searchDepth":531,"depth":531,"links":532},2,[533,537,538,539,540,541,542],{"id":67,"depth":531,"text":70,"children":534},[535],{"id":87,"depth":536,"text":90},3,{"id":247,"depth":531,"text":250},{"id":266,"depth":531,"text":269},{"id":293,"depth":531,"text":296},{"id":321,"depth":531,"text":324},{"id":415,"depth":531,"text":206},{"id":493,"depth":531,"text":496},"markdown","content:projects:ensemble.md","content","projects/ensemble.md","projects/ensemble","md",1776573205288]