From 0c4a83ebc868886fbf406d1f0f1b1c14d358e931 Mon Sep 17 00:00:00 2001
From: peterschmidt85 <andrey.cheptsov@gmail.com>
Date: Wed, 25 Feb 2026 14:51:39 +0100
Subject: [PATCH] Add robots.txt and structured data for SEO

Add docs/robots.txt pointing crawlers to the sitemap. Add JSON-LD
structured data to main.html: WebSite + Organization schema on the
homepage (helps Google show sitelinks and knowledge panel), and
BreadcrumbList schema on all other pages (helps Google show breadcrumb
trails in search results). Also update the Twitter social link to x.com.

AI Assistance: Claude

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 docs/overrides/main.html | 79 ++++++++++++++++++++++++++++++++++++++++
 docs/robots.txt          |  4 ++
 mkdocs.yml               |  2 +-
 3 files changed, 84 insertions(+), 1 deletion(-)
 create mode 100644 docs/robots.txt

diff --git a/docs/overrides/main.html b/docs/overrides/main.html
index ed5d28a8c..000ff9bec 100644
--- a/docs/overrides/main.html
+++ b/docs/overrides/main.html
@@ -5,6 +5,85 @@
 <link rel="preload" href="{{ 'assets/fonts/GeistMono-Variable.woff2' | url }}" as="font" type="font/woff2" crossorigin>
 <link rel="preload" href="{{ 'assets/fonts/GeistPixel-Square.woff2' | url }}" as="font" type="font/woff2" crossorigin>
 <link rel="preload" href="{{ 'assets/fonts/GeistPixel-Circle.woff2' | url }}" as="font" type="font/woff2" crossorigin>
+{#
+  Structured data (JSON-LD) for SEO.
+  - Homepage gets WebSite + Organization schema (helps Google show sitelinks and knowledge panel).
+  - All other pages get BreadcrumbList schema (helps Google show breadcrumb trails in results).
+  Breadcrumb URLs are resolved via _find_leaf_url because MkDocs nav sections don't have
+  their own URLs — we use the first descendant page's URL as a proxy.
+  Dedup by title to collapse nav levels duplicated by plugins (e.g. the blog plugin nests
+  "Blog" inside "Blog"). The current page is omitted when its title matches the last
+  ancestor (e.g. /examples/ is both the "Examples" section index and the page itself).
+#}
+{% macro _find_leaf_url(nav_item) -%}
+  {%- if nav_item.url -%}
+    /{{ nav_item.url }}
+  {%- elif nav_item.children -%}
+    {{ _find_leaf_url(nav_item.children | first) }}
+  {%- endif -%}
+{%- endmacro %}
+{% if page.is_homepage %}
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@graph": [
+    {
+      "@type": "WebSite",
+      "name": "dstack",
+      "url": "https://dstack.ai/"
+    },
+    {
+      "@type": "Organization",
+      "name": "dstack",
+      "url": "https://dstack.ai/",
+      "logo": "https://dstack.ai/assets/images/dstack-logo-notext.svg",
+      "sameAs": [
+        "https://github.com/dstackai/dstack",
+        "https://discord.gg/u8SmfwPpMd",
+        "https://x.com/dstackai",
+        "https://www.linkedin.com/company/dstackai"
+      ]
+    }
+  ]
+}
+</script>
+{% elif page.ancestors | length > 0 %}
+<script type="application/ld+json">
+{
+  "@context": "https://schema.org",
+  "@type": "BreadcrumbList",
+  "itemListElement": [
+    {
+      "@type": "ListItem",
+      "position": 1,
+      "name": "Home",
+      "item": "https://dstack.ai/"
+    }
+    {%- set pos = namespace(val=2) -%}
+    {%- set prev = namespace(title="") -%}
+    {%- for ancestor in page.ancestors | reverse -%}
+    {%- if ancestor.title != prev.title -%}
+    ,{
+      "@type": "ListItem",
+      "position": {{ pos.val }},
+      "name": "{{ ancestor.title | e }}",
+      "item": "https://dstack.ai{{ _find_leaf_url(ancestor) | trim }}"
+    }
+    {%- set pos.val = pos.val + 1 -%}
+    {%- endif -%}
+    {%- set prev.title = ancestor.title -%}
+    {%- endfor -%}
+    {%- if page.title != prev.title -%}
+    ,{
+      "@type": "ListItem",
+      "position": {{ pos.val }},
+      "name": "{{ page.title | e }}"
+    }
+    {%- endif -%}
+  ]
+}
+</script>
+{% endif %}
 {% endblock %}
 
 {% block container %}
diff --git a/docs/robots.txt b/docs/robots.txt
new file mode 100644
index 000000000..b38b74b55
--- /dev/null
+++ b/docs/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://dstack.ai/sitemap.xml
diff --git a/mkdocs.yml b/mkdocs.yml
index 1f9308451..58c176f3d 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -212,7 +212,7 @@ extra:
     - icon: /fontawesome/brands/discord
       link: https://discord.gg/u8SmfwPpMd
     - icon: /simple/x
-      link: https://twitter.com/dstackai
+      link: https://x.com/dstackai
     - icon: /fontawesome/brands/linkedin
       link: https://www.linkedin.com/company/dstackai
   status: