Added spark resource calculator

agbares · Feb 27, 2022 · 80d9596 · 80d9596
1 parent 66ed7a4
commit 80d9596
Show file tree

Hide file tree

Showing 6 changed files with 324 additions and 5 deletions.
diff --git a/blog/index.html b/blog/index.html
@@ -130,6 +130,20 @@
   <h1>Blogs</h1>
 </header>
 
+<article class="post-entry"> 
+  <header class="entry-header">
+    <h2>Spark Resource Calculator
+    </h2>
+  </header>
+  <section class="entry-content">
+    <p>After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.
+I decided to write a simple UI that calculates an executor’s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container’s entire memory space (e.g. executor memory, overhead, and off-heap).
+You can find the calculator by heading to agbares.com/spark-resource-calculator-ui</p>
+  </section>
+  <footer class="entry-footer"><span title='2022-02-26 22:53:30 -0800 PST'>February 26, 2022</span></footer>
+  <a class="entry-link" aria-label="post link to Spark Resource Calculator" href="https://agbares.com/blog/spark-resource-calculator/"></a>
+</article>
+
 <article class="post-entry"> 
   <header class="entry-header">
     <h2>Spark Executor Memory (Pyspark)

diff --git a/blog/index.xml b/blog/index.xml
@@ -6,7 +6,18 @@
     <description>Recent content in Blogs on Antonio Bares</description>
     <generator>Hugo -- gohugo.io</generator>
     <language>en-us</language>
-    <lastBuildDate>Sun, 06 Feb 2022 17:35:01 -0800</lastBuildDate><atom:link href="https://agbares.com/blog/index.xml" rel="self" type="application/rss+xml" />
+    <lastBuildDate>Sat, 26 Feb 2022 22:53:30 -0800</lastBuildDate><atom:link href="https://agbares.com/blog/index.xml" rel="self" type="application/rss+xml" />
+    <item>
+      <title>Spark Resource Calculator</title>
+      <link>https://agbares.com/blog/spark-resource-calculator/</link>
+      <pubDate>Sat, 26 Feb 2022 22:53:30 -0800</pubDate>
+
+      <guid>https://agbares.com/blog/spark-resource-calculator/</guid>
+      <description>After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.
+I decided to write a simple UI that calculates an executor&amp;rsquo;s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container&amp;rsquo;s entire memory space (e.g. executor memory, overhead, and off-heap).
+You can find the calculator by heading to agbares.com/spark-resource-calculator-ui</description>
+    </item>
+
     <item>
       <title>Spark Executor Memory (Pyspark)</title>
       <link>https://agbares.com/blog/spark-executor-memory/</link>

diff --git a/blog/spark-resource-calculator/index.html b/blog/spark-resource-calculator/index.html
@@ -0,0 +1,266 @@
+<!DOCTYPE html>
+<html lang="en" dir="auto">
+
+<head><meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+<meta name="robots" content="index, follow">
+<title>Spark Resource Calculator | Antonio Bares</title>
+<meta name="keywords" content="" />
+<meta name="description" content="After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.
+I decided to write a simple UI that calculates an executor&rsquo;s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container&rsquo;s entire memory space (e.g. executor memory, overhead, and off-heap).
+You can find the calculator by heading to agbares.com/spark-resource-calculator-ui">
+<meta name="author" content="">
+<link rel="canonical" href="https://agbares.com/blog/spark-resource-calculator/" />
+<link crossorigin="anonymous" href="/assets/css/stylesheet.min.c88963fe2d79462000fd0fb1b3737783c32855d340583e4523343f8735c787f0.css" integrity="sha256-yIlj/i15RiAA/Q&#43;xs3N3g8MoVdNAWD5FIzQ/hzXHh/A=" rel="preload stylesheet" as="style">
+<script defer crossorigin="anonymous" src="/assets/js/highlight.min.7680afc38aa6b15ddf158a4f3780b7b1f7dde7e91d26f073e6229bb7a0793c92.js" integrity="sha256-doCvw4qmsV3fFYpPN4C3sffd5&#43;kdJvBz5iKbt6B5PJI="
+    onload="hljs.initHighlightingOnLoad();"></script>
+<link rel="icon" href="https://agbares.com/favicon.ico">
+<link rel="icon" type="image/png" sizes="16x16" href="https://agbares.com/favicon-16x16.png">
+<link rel="icon" type="image/png" sizes="32x32" href="https://agbares.com/favicon-32x32.png">
+<link rel="apple-touch-icon" href="https://agbares.com/apple-touch-icon.png">
+<link rel="mask-icon" href="https://agbares.com/safari-pinned-tab.svg">
+<meta name="theme-color" content="#2e2e33">
+<meta name="msapplication-TileColor" content="#2e2e33">
+<meta name="generator" content="Hugo 0.92.1" />
+<noscript>
+    <style>
+        #theme-toggle,
+        .top-link {
+            display: none;
+        }
+
+    </style>
+    <style>
+        @media (prefers-color-scheme: dark) {
+            :root {
+                --theme: rgb(29, 30, 32);
+                --entry: rgb(46, 46, 51);
+                --primary: rgb(218, 218, 219);
+                --secondary: rgb(155, 156, 157);
+                --tertiary: rgb(65, 66, 68);
+                --content: rgb(196, 196, 197);
+                --hljs-bg: rgb(46, 46, 51);
+                --code-bg: rgb(55, 56, 62);
+                --border: rgb(51, 51, 51);
+            }
+
+            .list {
+                background: var(--theme);
+            }
+
+            .list:not(.dark)::-webkit-scrollbar-track {
+                background: 0 0;
+            }
+
+            .list:not(.dark)::-webkit-scrollbar-thumb {
+                border-color: var(--theme);
+            }
+        }
+
+    </style>
+</noscript><meta property="og:title" content="Spark Resource Calculator" />
+<meta property="og:description" content="After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.
+I decided to write a simple UI that calculates an executor&rsquo;s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container&rsquo;s entire memory space (e.g. executor memory, overhead, and off-heap).
+You can find the calculator by heading to agbares.com/spark-resource-calculator-ui" />
+<meta property="og:type" content="article" />
+<meta property="og:url" content="https://agbares.com/blog/spark-resource-calculator/" /><meta property="article:section" content="blog" />
+<meta property="article:published_time" content="2022-02-26T22:53:30-08:00" />
+<meta property="article:modified_time" content="2022-02-26T22:53:30-08:00" />
+
+<meta name="twitter:card" content="summary"/>
+<meta name="twitter:title" content="Spark Resource Calculator"/>
+<meta name="twitter:description" content="After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.
+I decided to write a simple UI that calculates an executor&rsquo;s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container&rsquo;s entire memory space (e.g. executor memory, overhead, and off-heap).
+You can find the calculator by heading to agbares.com/spark-resource-calculator-ui"/>
+
+
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@type": "BreadcrumbList",
+  "itemListElement": [, 
+    {
+      "@type": "ListItem",
+      "position":  2 ,
+      "name": "Blogs",
+      "item": "https://agbares.com/blog/"
+    }, 
+    {
+      "@type": "ListItem",
+      "position":  3 ,
+      "name": "Spark Resource Calculator",
+      "item": "https://agbares.com/blog/spark-resource-calculator/"
+    }
+  ]
+}
+</script>
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@type": "BlogPosting",
+  "headline": "Spark Resource Calculator",
+  "name": "Spark Resource Calculator",
+  "description": "After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.\nI decided to write a simple UI that calculates an executor\u0026rsquo;s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container\u0026rsquo;s entire memory space (e.g. executor memory, overhead, and off-heap).\nYou can find the calculator by heading to agbares.com/spark-resource-calculator-ui",
+  "keywords": [
+
+  ],
+  "articleBody": "After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.\nI decided to write a simple UI that calculates an executor’s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container’s entire memory space (e.g. executor memory, overhead, and off-heap).\nYou can find the calculator by heading to agbares.com/spark-resource-calculator-ui\n",
+  "wordCount" : "67",
+  "inLanguage": "en",
+  "datePublished": "2022-02-26T22:53:30-08:00",
+  "dateModified": "2022-02-26T22:53:30-08:00",
+  "mainEntityOfPage": {
+    "@type": "WebPage",
+    "@id": "https://agbares.com/blog/spark-resource-calculator/"
+  },
+  "publisher": {
+    "@type": "Organization",
+    "name": "Antonio Bares",
+    "logo": {
+      "@type": "ImageObject",
+      "url": "https://agbares.com/favicon.ico"
+    }
+  }
+}
+</script>
+</head>
+
+<body class="" id="top">
+<script>
+    if (localStorage.getItem("pref-theme") === "dark") {
+        document.body.classList.add('dark');
+    } else if (localStorage.getItem("pref-theme") === "light") {
+        document.body.classList.remove('dark')
+    } else if (window.matchMedia('(prefers-color-scheme: dark)').matches) {
+        document.body.classList.add('dark');
+    }
+
+</script>
+
+<header class="header">
+    <nav class="nav">
+        <div class="logo">
+            <a href="https://agbares.com" accesskey="h" title="Antonio Bares (Alt + H)">Antonio Bares</a>
+            <span class="logo-switches">
+                <button id="theme-toggle" accesskey="t" title="(Alt + T)">
+                    <svg id="moon" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"
+                        fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
+                        stroke-linejoin="round">
+                        <path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z"></path>
+                    </svg>
+                    <svg id="sun" xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24"
+                        fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round"
+                        stroke-linejoin="round">
+                        <circle cx="12" cy="12" r="5"></circle>
+                        <line x1="12" y1="1" x2="12" y2="3"></line>
+                        <line x1="12" y1="21" x2="12" y2="23"></line>
+                        <line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
+                        <line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
+                        <line x1="1" y1="12" x2="3" y2="12"></line>
+                        <line x1="21" y1="12" x2="23" y2="12"></line>
+                        <line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
+                        <line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
+                    </svg>
+                </button>
+            </span>
+        </div>
+        <ul id="menu">
+        </ul>
+    </nav>
+</header>
+<main class="main">
+
+<article class="post-single">
+  <header class="post-header">
+
+    <h1 class="post-title">
+      Spark Resource Calculator
+    </h1>
+    <div class="post-meta"><span title='2022-02-26 22:53:30 -0800 PST'>February 26, 2022</span>
+
+</div>
+  </header> 
+  <div class="post-content"><p>After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.</p>
+<p>I decided to write a simple UI that calculates an executor&rsquo;s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container&rsquo;s entire memory space (e.g. executor memory, overhead, and off-heap).</p>
+<p>You can find the calculator by heading to <a href="https://agbares.com/spark-resource-calculator-ui">agbares.com/spark-resource-calculator-ui</a></p>
+
+
+  </div>
+
+  <footer class="post-footer">
+  </footer>
+</article>
+    </main>
+
+<footer class="footer">
+    <span>&copy; 2022 <a href="https://agbares.com">Antonio Bares</a></span>
+    <span>
+        Powered by
+        <a href="https://gohugo.io/" rel="noopener noreferrer" target="_blank">Hugo</a> &
+        <a href="https://git.io/hugopapermod" rel="noopener" target="_blank">PaperMod</a>
+    </span>
+</footer>
+<a href="#top" aria-label="go to top" title="Go to Top (Alt + G)" class="top-link" id="top-link" accesskey="g">
+    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 12 6" fill="currentColor">
+        <path d="M12 6H0l6-6z" />
+    </svg>
+</a>
+
+<script>
+    let menu = document.getElementById('menu')
+    if (menu) {
+        menu.scrollLeft = localStorage.getItem("menu-scroll-position");
+        menu.onscroll = function () {
+            localStorage.setItem("menu-scroll-position", menu.scrollLeft);
+        }
+    }
+
+    document.querySelectorAll('a[href^="#"]').forEach(anchor => {
+        anchor.addEventListener("click", function (e) {
+            e.preventDefault();
+            var id = this.getAttribute("href").substr(1);
+            if (!window.matchMedia('(prefers-reduced-motion: reduce)').matches) {
+                document.querySelector(`[id='${decodeURIComponent(id)}']`).scrollIntoView({
+                    behavior: "smooth"
+                });
+            } else {
+                document.querySelector(`[id='${decodeURIComponent(id)}']`).scrollIntoView();
+            }
+            if (id === "top") {
+                history.replaceState(null, null, " ");
+            } else {
+                history.pushState(null, null, `#${id}`);
+            }
+        });
+    });
+
+</script>
+<script>
+    var mybutton = document.getElementById("top-link");
+    window.onscroll = function () {
+        if (document.body.scrollTop > 800 || document.documentElement.scrollTop > 800) {
+            mybutton.style.visibility = "visible";
+            mybutton.style.opacity = "1";
+        } else {
+            mybutton.style.visibility = "hidden";
+            mybutton.style.opacity = "0";
+        }
+    };
+
+</script>
+<script>
+    document.getElementById("theme-toggle").addEventListener("click", () => {
+        if (document.body.className.includes("dark")) {
+            document.body.classList.remove('dark');
+            localStorage.setItem("pref-theme", 'light');
+        } else {
+            document.body.classList.add('dark');
+            localStorage.setItem("pref-theme", 'dark');
+        }
+    })
+
+</script>
+</body>
+
+</html>
diff --git a/index.html b/index.html
@@ -129,7 +129,7 @@
         <h1>Hello</h1>
     </header>
     <section class="entry-content">
-        <p>Welcome to personal my writings on some of my learnings. I believe writing them in a way to teach others is a great way to solidfy my knowledge. Feel free to reach out to me at <a href="mailto:[email protected]">[email protected]</a></p>
+        <p>Welcome to my personal writings on some of my learnings. Feel free to reach out to me at <a href="mailto:[email protected]">[email protected]</a></p>
     </section>
     <footer class="entry-footer">
         <div class="social-icons">
@@ -146,6 +146,20 @@ <h1>Hello</h1>
     </footer>
 </article>
 
+<article class="post-entry"> 
+  <header class="entry-header">
+    <h2>Spark Resource Calculator
+    </h2>
+  </header>
+  <section class="entry-content">
+    <p>After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.
+I decided to write a simple UI that calculates an executor’s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container’s entire memory space (e.g. executor memory, overhead, and off-heap).
+You can find the calculator by heading to agbares.com/spark-resource-calculator-ui</p>
+  </section>
+  <footer class="entry-footer"><span title='2022-02-26 22:53:30 -0800 PST'>February 26, 2022</span></footer>
+  <a class="entry-link" aria-label="post link to Spark Resource Calculator" href="https://agbares.com/blog/spark-resource-calculator/"></a>
+</article>
+
 <article class="post-entry"> 
   <header class="entry-header">
     <h2>Spark Executor Memory (Pyspark)

diff --git a/index.xml b/index.xml
@@ -6,7 +6,18 @@
     <description>Recent content on Antonio Bares</description>
     <generator>Hugo -- gohugo.io</generator>
     <language>en-us</language>
-    <lastBuildDate>Sun, 06 Feb 2022 17:35:01 -0800</lastBuildDate><atom:link href="https://agbares.com/index.xml" rel="self" type="application/rss+xml" />
+    <lastBuildDate>Sat, 26 Feb 2022 22:53:30 -0800</lastBuildDate><atom:link href="https://agbares.com/index.xml" rel="self" type="application/rss+xml" />
+    <item>
+      <title>Spark Resource Calculator</title>
+      <link>https://agbares.com/blog/spark-resource-calculator/</link>
+      <pubDate>Sat, 26 Feb 2022 22:53:30 -0800</pubDate>
+
+      <guid>https://agbares.com/blog/spark-resource-calculator/</guid>
+      <description>After optimizing a few spark jobs, I realized that calculating the executor memory space is an extremely manual process.
+I decided to write a simple UI that calculates an executor&amp;rsquo;s on-heap memory space (e.g. spark/executor/storage memory). I might extend this in the future to also include the executor container&amp;rsquo;s entire memory space (e.g. executor memory, overhead, and off-heap).
+You can find the calculator by heading to agbares.com/spark-resource-calculator-ui</description>
+    </item>
+
     <item>
       <title>Spark Executor Memory (Pyspark)</title>
       <link>https://agbares.com/blog/spark-executor-memory/</link>

diff --git a/sitemap.xml b/sitemap.xml
@@ -3,10 +3,13 @@
   xmlns:xhtml="http://www.w3.org/1999/xhtml">
   <url>
     <loc>https://agbares.com/</loc>
-    <lastmod>2022-02-06T17:35:01-08:00</lastmod>
+    <lastmod>2022-02-26T22:53:30-08:00</lastmod>
   </url><url>
     <loc>https://agbares.com/blog/</loc>
-    <lastmod>2022-02-06T17:35:01-08:00</lastmod>
+    <lastmod>2022-02-26T22:53:30-08:00</lastmod>
+  </url><url>
+    <loc>https://agbares.com/blog/spark-resource-calculator/</loc>
+    <lastmod>2022-02-26T22:53:30-08:00</lastmod>
   </url><url>
     <loc>https://agbares.com/blog/spark-executor-memory/</loc>
     <lastmod>2022-02-06T17:35:01-08:00</lastmod>