rock/homelab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: patch bootstrap cluster with correct config

Browse Source
This commit is contained in:
Story Crater Bot
2026-07-25 07:09:45 -07:00
parent 95ae933489
commit bf67d2d9de
7 changed files with 170 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -55,3 +55,5 @@ skills-lock.json
.DS_Store
CLAUDE.md
docs/
k8s/argocd/seed-repo-secret.yaml
Makefile
+128 -209
View File
@@ -1,21 +1,21 @@
# ── Node IPs ──────────────────────────────────────────────────────────────────
# CP_IP has a default. All W{N}_IP variables are expected to be exported from
# ~/.zshrc (e.g. export W1_IP=192.168.1.162). No guards — assumed always set.
CP_IP ?= 192.168.1.213
export CP_IP
# ── Node IPs (3-CP HA topology) ───────────────────────────────────────────────
CP1_IP := 192.168.1.166 # talos-cp-1
CP2_IP := 192.168.1.213 # talos-cp-2 (storage: 3 disks)
CP3_IP := 192.168.1.162 # talos-cp-3
CP_VIP := 192.168.1.166 # controlplane VIP (currently .166)
# ── Paths ─────────────────────────────────────────────────────────────────────
TALOSCONFIG := cluster-config/coreconfig
CP_CONFIG := cluster-config/controlplane.yaml
SECRETS := cluster-config/secrets.yaml
TALOSCONFIG := cluster-config/talosconfig
CP1_CONFIG := cluster-config/talos-cp-1.yaml
CP2_CONFIG := cluster-config/talos-cp-2.yaml
CP3_CONFIG := cluster-config/talos-cp-3.yaml
KUBECONFIG := cluster-config/kubeconfig
CLUSTER_NAME := homelab-cluster
CP_ENDPOINT := https://$(CP_IP):6443
TALOS_IMAGE := factory.core.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:v1.13.3
CLUSTER_NAME := homelab
CP_ENDPOINT := https://$(CP_VIP):6443
TALOSCTL := corectl --coreconfig $(TALOSCONFIG)
# Use talosctl (not corectl). Needs TALOSCONFIG env var pointing to talosconfig file.
TALOSCTL := talosctl
KUBECTL := kubectl --kubeconfig $(KUBECONFIG)
# Derive IP and config from worker number N (used by generic targets).
@@ -28,236 +28,157 @@ W_CONFIG = cluster-config/worker-$(N).yaml
# ── Help ──────────────────────────────────────────────────────────────────────
.PHONY: help
help:
@echo "Homelab cluster — available targets"
@echo "Homelab cluster (3-CP HA: .166/.213/.163) — available targets"
@echo ""
@echo " Status"
@echo " Status & Services"
@echo " nodes kubectl get nodes"
@echo " status-cp core node overview (control plane)"
@echo " status-w1 core node overview (worker-1)"
@echo " services-cp list core services (control plane)"
@echo " services-w1 list core services (worker-1)"
@echo " status-all etcd members on all 3 CPs"
@echo " status-cp1/2/3 etcd members on specific CP"
@echo " services-cp1/2/3 list Talos services on specific CP"
@echo ""
@echo " Logs"
@echo " logs-cp stream kubelet logs (control plane)"
@echo " logs-w1 stream kubelet logs (worker-1)"
@echo " dmesg-cp kernel dmesg (control plane)"
@echo " dmesg-w1 kernel dmesg (worker-1)"
@echo " log-svc-cp stream a service log (control plane) SVC=<name>"
@echo " log-svc-w1 stream a service log (worker-1) SVC=<name>"
@echo " logs-cp1/2/3 stream kubelet logs from CP{1,2,3}"
@echo " dmesg-cp1/2/3 stream kernel dmesg from CP{1,2,3}"
@echo " log-svc-cp1/2/3 stream service logs (SVC=<name>)"
@echo ""
@echo " Config"
@echo " gen-config regenerate controlplane.yaml + worker-N.yaml from secrets"
@echo " apply-cp apply controlplane.yaml to CP node (live cluster)"
@echo " apply-w1 apply cluster-config/worker-1.yaml to worker-1"
@echo " apply-w1-insecure first-time apply to worker-1 (no certs yet)"
@echo " apply-worker apply cluster-config/worker-N.yaml N=<num> W<N>_IP=<ip>"
@echo " apply-worker-new first-time apply (--insecure) N=<num> W<N>_IP=<ip>"
@echo " Config Apply"
@echo " apply-all apply configs to all 3 CPs (talos-cp-{1,2,3}.yaml)"
@echo " apply-cp1/2/3 apply config to specific CP"
@echo ""
@echo " Upgrade"
@echo " upgrade-cp upgrade Talos on control plane"
@echo " upgrade-w1 upgrade Talos on worker-1"
@echo " upgrade-worker upgrade any worker N=<num> W<N>_IP=<ip>"
@echo ""
@echo " Shutdown / Reboot"
@echo " shutdown-cluster graceful full shutdown (drain w1 → off w1 → off cp)"
@echo " shutdown-cp shut down control plane only"
@echo " shutdown-w1 shut down worker-1 only"
@echo " shutdown-worker shut down any worker N=<num> W<N>_IP=<ip>"
@echo " reboot-cp reboot control plane"
@echo " reboot-w1 reboot worker-1"
@echo " reboot-worker reboot any worker N=<num> W<N>_IP=<ip>"
@echo ""
@echo " Inspect (node filesystem)"
@echo " node-ls <ip> <path> list files on a node"
@echo " node-read <ip> <path> read a file on a node"
@echo ""
@echo " Maintenance"
@echo " clean-pods delete Evicted/Failed/Terminating pods cluster-wide"
@echo " Reboot"
@echo " reboot-all reboot all 3 CPs"
@echo " reboot-cp1/2/3 reboot specific CP"
@echo ""
@echo " Port-forwards"
@echo " pf-grafana localhost:3000 → Grafana"
@echo " pf-minio localhost:9001 → MinIO console / localhost:9000 → S3 API"
@echo " pf-loki localhost:3100 → Loki HTTP API"
@echo " pf-portainer localhost:9000 → Portainer UI (dashboard ns)"
@echo " pf-prometheus localhost:9090 → Prometheus UI (monitoring ns)"
@echo " pf-longhorn localhost:8080 → Longhorn UI"
@echo " pf-iam localhost:7000 → Authentik IAM (when deployed)"
@echo " pf-prometheus localhost:9090 → Prometheus UI"
@echo ""
@echo " CLI"
@echo " cli build core-cli and install to ~/.local/bin/core"
@echo ""
@echo " Variables"
@echo " CP_IP (default: 192.168.1.160)"
@echo " W1_IP (export from ~/.zshrc — e.g. export W1_IP=192.168.1.162)"
@echo " N (required for generic targets — worker number, e.g. N=2)"
@echo " W<N>_IP (export from ~/.zshrc — e.g. export W2_IP=192.168.1.163)"
@echo " SVC (required for log-svc-* targets, e.g. SVC=kubelet)"
@echo " IPs"
@echo " CP1 (talos-cp-1): $(CP1_IP) — NVMe, wg0/wg1, VIP"
@echo " CP2 (talos-cp-2): $(CP2_IP) — 3 Longhorn disks"
@echo " CP3 (talos-cp-3): $(CP3_IP) — NVMe"
# ── Status ────────────────────────────────────────────────────────────────────
.PHONY: nodes
nodes:
$(KUBECTL) get nodes -o wide
.PHONY: status-all
status-all: status-cp1 status-cp2 status-cp3
.PHONY: status-cp1
status-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) etcd members
.PHONY: status-cp2
status-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) etcd members
.PHONY: status-cp3
status-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) etcd members
.PHONY: status-cp
status-cp:
$(TALOSCTL) --nodes $(CP_IP) get members
status-cp: status-all
.PHONY: status-w1
status-w1:
$(TALOSCTL) --nodes $(W1_IP) get members
.PHONY: services-cp1
services-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) service
.PHONY: services-cp
services-cp:
$(TALOSCTL) --nodes $(CP_IP) service
.PHONY: services-cp2
services-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) service
.PHONY: services-w1
services-w1:
$(TALOSCTL) --nodes $(W1_IP) service
.PHONY: services-cp3
services-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) service
# ── Logs ──────────────────────────────────────────────────────────────────────
.PHONY: logs-cp
logs-cp:
$(TALOSCTL) --nodes $(CP_IP) logs kubelet -f
# ── Logs (3-CP) ───────────────────────────────────────────────────────────────
.PHONY: logs-cp1
logs-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) logs kubelet -f
.PHONY: logs-w1
logs-w1:
$(TALOSCTL) --nodes $(W1_IP) logs kubelet -f
.PHONY: logs-cp2
logs-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) logs kubelet -f
.PHONY: dmesg-cp
dmesg-cp:
$(TALOSCTL) --nodes $(CP_IP) dmesg --follow
.PHONY: logs-cp3
logs-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) logs kubelet -f
.PHONY: dmesg-w1
dmesg-w1:
$(TALOSCTL) --nodes $(W1_IP) dmesg --follow
.PHONY: dmesg-cp1
dmesg-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) dmesg --follow
# Usage: make log-svc-cp SVC=etcd
.PHONY: log-svc-cp
log-svc-cp:
.PHONY: dmesg-cp2
dmesg-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) dmesg --follow
.PHONY: dmesg-cp3
dmesg-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) dmesg --follow
# Usage: make log-svc-cp1 SVC=etcd
.PHONY: log-svc-cp1
log-svc-cp1:
ifndef SVC
$(error SVC is not set — run: make log-svc-cp SVC=<service-name>)
$(error SVC is not set — run: make log-svc-cp1 SVC=<service-name>)
endif
$(TALOSCTL) --nodes $(CP_IP) logs $(SVC) -f
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) logs $(SVC) -f
.PHONY: log-svc-w1
log-svc-w1:
.PHONY: log-svc-cp2
log-svc-cp2:
ifndef SVC
$(error SVC is not set — run: make log-svc-w1 SVC=<service-name>)
$(error SVC is not set — run: make log-svc-cp2 SVC=<service-name>)
endif
$(TALOSCTL) --nodes $(W1_IP) logs $(SVC) -f
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) logs $(SVC) -f
# ── Config generation ─────────────────────────────────────────────────────────
.PHONY: gen-config
gen-config:
corectl gen config $(CLUSTER_NAME) $(CP_ENDPOINT) \
--with-secrets $(SECRETS) \
--output-dir cluster-config/ \
--force
.PHONY: log-svc-cp3
log-svc-cp3:
ifndef SVC
$(error SVC is not set — run: make log-svc-cp3 SVC=<service-name>)
endif
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) logs $(SVC) -f
# ── Config Apply (3-CP) ───────────────────────────────────────────────────────
.PHONY: apply-all
apply-all: apply-cp1 apply-cp2 apply-cp3
@echo "✓ All 3 control planes configured"
.PHONY: apply-cp1
apply-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) apply-config -f $(CP1_CONFIG)
.PHONY: apply-cp2
apply-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) apply-config -f $(CP2_CONFIG)
.PHONY: apply-cp3
apply-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) apply-config -f $(CP3_CONFIG)
# ── Config apply ──────────────────────────────────────────────────────────────
.PHONY: apply-cp
apply-cp:
$(TALOSCTL) apply-config \
--nodes $(CP_IP) \
--file $(CP_CONFIG)
apply-cp: apply-all
.PHONY: apply-w1
apply-w1:
$(TALOSCTL) apply-config \
--nodes $(W1_IP) \
--file cluster-config/worker-1.yaml
.PHONY: reboot-all
reboot-all: reboot-cp1 reboot-cp2 reboot-cp3
@echo "✓ All 3 control planes rebooting"
# First-time apply to worker-1 (no certs yet)
.PHONY: apply-w1-insecure
apply-w1-insecure:
$(TALOSCTL) apply-config \
--nodes $(W1_IP) \
--file cluster-config/worker-1.yaml \
--insecure
.PHONY: reboot-cp1
reboot-cp1:
$(TALOSCTL) -n $(CP1_IP) --endpoints $(CP1_IP) reboot
# Generic targets — derive both IP and config from N.
# Usage: make apply-worker N=2 W2_IP=192.168.1.162
# make apply-worker N=3 W3_IP=192.168.1.163
.PHONY: apply-worker
apply-worker:
ifndef N
$(error N is not set — run: make apply-worker N=<num> W<N>_IP=<ip>)
endif
$(TALOSCTL) apply-config \
--nodes $(W_IP) \
--file $(W_CONFIG)
.PHONY: reboot-cp2
reboot-cp2:
$(TALOSCTL) -n $(CP2_IP) --endpoints $(CP2_IP) reboot
.PHONY: apply-worker-new
apply-worker-new:
ifndef N
$(error N is not set — run: make apply-worker-new N=<num> W<N>_IP=<ip>)
endif
$(TALOSCTL) apply-config \
--nodes $(W_IP) \
--file $(W_CONFIG) \
--insecure
# ── Upgrade ───────────────────────────────────────────────────────────────────
.PHONY: upgrade-cp
upgrade-cp:
$(TALOSCTL) upgrade \
--nodes $(CP_IP) \
--image $(TALOS_IMAGE) \
--preserve
.PHONY: upgrade-w1
upgrade-w1:
$(TALOSCTL) upgrade \
--nodes $(W1_IP) \
--image $(TALOS_IMAGE) \
--preserve
# Usage: make upgrade-worker N=2 W2_IP=192.168.1.162
.PHONY: upgrade-worker
upgrade-worker:
ifndef N
$(error N is not set — run: make upgrade-worker N=<num> W<N>_IP=<ip>)
endif
$(TALOSCTL) upgrade \
--nodes $(W_IP) \
--image $(TALOS_IMAGE) \
--preserve
# ── Shutdown / Reboot ─────────────────────────────────────────────────────────
# Full cluster: drain workers first so pods stop cleanly, then workers off,
# then CP last (etcd must be the final process to stop).
.PHONY: shutdown-cluster
shutdown-cluster:
@echo "--- draining core-worker-1 ---"
$(KUBECTL) drain core-worker-1 --ignore-daemonsets --delete-emptydir-data
@echo "--- shutting down worker-1 ---"
$(TALOSCTL) --nodes $(W1_IP) shutdown
@echo "--- shutting down control plane (last) ---"
$(TALOSCTL) --nodes $(CP_IP) shutdown
.PHONY: shutdown-cp
shutdown-cp:
$(TALOSCTL) --nodes $(CP_IP) shutdown
.PHONY: shutdown-w1
shutdown-w1:
$(TALOSCTL) --nodes $(W1_IP) shutdown
# Usage: make shutdown-worker N=2 W2_IP=192.168.1.162
.PHONY: shutdown-worker
shutdown-worker:
ifndef N
$(error N is not set — run: make shutdown-worker N=<num> W<N>_IP=<ip>)
endif
$(TALOSCTL) --nodes $(W_IP) shutdown
.PHONY: reboot-cp3
reboot-cp3:
$(TALOSCTL) -n $(CP3_IP) --endpoints $(CP3_IP) reboot
.PHONY: reboot-cp
reboot-cp:
$(TALOSCTL) --nodes $(CP_IP) reboot
.PHONY: reboot-w1
reboot-w1:
$(TALOSCTL) --nodes $(W1_IP) reboot
reboot-cp: reboot-all
# Usage: make reboot-worker N=2 W2_IP=192.168.1.162
.PHONY: reboot-worker
@@ -268,18 +189,16 @@ endif
$(TALOSCTL) --nodes $(W_IP) reboot
# ── Inspect ───────────────────────────────────────────────────────────────────
# Positional args: make node-ls 192.168.1.160 /etc/kubernetes/manifests
# $(word 2/3, $(MAKECMDGOALS)) captures the extra words; the % rule absorbs
# them so Make doesn't error with "No rule to make target".
# Positional args: make node-ls 192.168.1.166 /etc/kubernetes/manifests
.PHONY: node-ls
node-ls:
$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
$(TALOSCTL) -n $(word 2,$(MAKECMDGOALS)) --endpoints $(word 2,$(MAKECMDGOALS)) ls $(word 3,$(MAKECMDGOALS))
.PHONY: node-read
node-read:
$(TALOSCTL) --nodes $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
$(TALOSCTL) -n $(word 2,$(MAKECMDGOALS)) --endpoints $(word 2,$(MAKECMDGOALS)) read $(word 3,$(MAKECMDGOALS))
# Absorb positional arguments passed to node-ls / node-read
# Absorb positional arguments
%:
@:
k8s/bootstrap/phase1-storage/longhorn-values.yaml
+1
View File
@@ -26,6 +26,7 @@ persistence:
# CSI plugin must tolerate control-plane taints
csi:
iscsiadmPath: /usr/sbin/iscsiadm
kubeletRootDir: /var/lib/kubelet
attacherReplicaCount: 3
provisionerReplicaCount: 3
terraform/main.tf
+2 -11
View File
@@ -11,7 +11,8 @@ locals {
kube_proxy_img = "registry.k8s.io/kube-proxy:${var.kubernetes_version}"
scheduler_img = "registry.k8s.io/kube-scheduler:${var.kubernetes_version}"
factory_image = "factory.talos.dev/installer/613e1592b2da41ae5e265e8789429f22e121aab91cb4deb6bc3c0b6262961245:${var.talos_version}"
# Use Image Factory generated URL with Longhorn extensions (iscsi-tools, util-linux-tools)
factory_image = data.talos_image_factory_urls.longhorn_installer.urls.installer
}
# Control plane machine configurations
@@ -29,16 +30,6 @@ resource "local_file" "controlplane_configs" {
lan_ip = each.value.lan_ip
lan_subnet = each.value.lan_subnet
lan_gateway = each.value.lan_gateway
wg0_ip = each.value.wg0_ip
wg0_subnet = each.value.wg0_subnet
wg0_port = each.value.wg0_port
wg0_private_key = each.value.wg0_private_key
wg0_peers = each.value.wg0_peers
wg1_ip = each.value.wg1_ip
wg1_subnet = each.value.wg1_subnet
wg1_port = each.value.wg1_port
wg1_private_key = each.value.wg1_private_key
wg1_peers = each.value.wg1_peers
kubelet_image = local.kubelet_image
cluster_dns_ip = local.cluster_dns_ip
install_disk = each.value.install_disk
terraform/talos_schematic.tf
+31
View File
@@ -0,0 +1,31 @@
# Talos Image Factory schematic with Longhorn dependencies
resource "talos_image_factory_schematic" "longhorn" {
schematic = jsonencode({
customization = {
systemExtensions = {
officialExtensions = [
"siderolabs/iscsi-tools",
"siderolabs/util-linux-tools",
]
}
}
})
}
# Generate installer image URL for the schematic
data "talos_image_factory_urls" "longhorn_installer" {
talos_version = var.talos_version
schematic_id = talos_image_factory_schematic.longhorn.id
platform = "metal"
}
# Output the schematic ID and installer URL for reference
output "talos_schematic_id" {
value = talos_image_factory_schematic.longhorn.id
description = "Talos Image Factory schematic ID with Longhorn dependencies"
}
output "talos_installer_url" {
value = data.talos_image_factory_urls.longhorn_installer.urls.installer
description = "Talos installer image URL with iscsi-tools and util-linux-tools"
}
terraform/templates/controlplane.tftpl
+2 -44
View File
@@ -10,9 +10,6 @@ machine:
key: ${ca_key}
certSANs:
- ${lan_ip}
%{ if wg0_ip != null ~}
- ${wg0_ip}
%{ endif ~}
%{ for san in cloudflare_talos_sans ~}
- ${san}
%{ endfor ~}
@@ -28,39 +25,6 @@ machine:
dhcp: false
dhcpOptions:
ipv6: false
%{ if wg0_ip != null ~}
- interface: wg0
addresses:
- ${wg0_ip}/24
wireguard:
privateKey: "${wg0_private_key}"
listenPort: ${wg0_port}
peers:
%{ for peer in wg0_peers ~}
- publicKey: "${peer.public_key}"
allowedIPs:
%{ for ip in peer.allowed_ips ~}
- ${ip}
%{ endfor ~}
%{ endfor ~}
%{ endif ~}
%{ if wg1_ip != null ~}
- interface: wg1
addresses:
- ${wg1_ip}/24
wireguard:
privateKey: "${wg1_private_key}"
listenPort: ${wg1_port}
peers:
%{ for peer in wg1_peers ~}
- publicKey: "${peer.public_key}"
allowedIPs:
%{ for ip in peer.allowed_ips ~}
- ${ip}
%{ endfor ~}
persistentKeepaliveInterval: ${peer.persistent_keepalive_secs}s
%{ endfor ~}
%{ endif ~}
nameservers:
%{ for ns in dns_servers ~}
- ${ns}
@@ -104,8 +68,7 @@ machine:
topology.kubernetes.io/zone: ${zone}
%{ if !allow_scheduling ~}
# Dedicated control plane — re-apply the control-plane taint that
# allowSchedulingOnControlPlanes=true removed cluster-wide. Only nodes with
# allow_scheduling=true (talos-cp-1 / .213) stay schedulable.
# allowSchedulingOnControlPlanes=true removed cluster-wide.
nodeTaints:
node-role.kubernetes.io/control-plane: ":NoSchedule"
%{ endif ~}
@@ -144,9 +107,6 @@ cluster:
apiServer:
certSANs:
- ${controlplane_ip}
%{ if wg0_ip != null ~}
- ${wg0_ip}
%{ endif ~}
- ${lan_ip}
%{ for san in cloudflare_apiserver_sans ~}
- ${san}
@@ -188,9 +148,7 @@ cluster:
disabled: true
service: {}
etcd:
# Advertise/peer etcd on the LAN so all control planes can reach each other.
# Without this, Talos may pick the WireGuard IP (10.6.0.1), which the
# LAN-only control planes can't route to — new members get stuck as learners.
# Pin etcd peer/advertise addresses to the LAN subnet explicitly.
advertisedSubnets:
- 192.168.1.0/24
ca:
terraform/variables.tf
+4 -21
View File
@@ -102,27 +102,10 @@ variable "secretbox_encryption_secret" {
variable "controlplane_configs" {
type = map(object({
hostname = string
lan_ip = string
lan_subnet = string
lan_gateway = string
wg0_ip = optional(string)
wg0_subnet = optional(string)
wg0_port = optional(number)
wg0_private_key = optional(string)
wg0_peers = optional(list(object({
public_key = string
allowed_ips = list(string)
})), [])
wg1_ip = optional(string)
wg1_subnet = optional(string)
wg1_port = optional(number)
wg1_private_key = optional(string)
wg1_peers = optional(list(object({
public_key = string
allowed_ips = list(string)
persistent_keepalive_secs = number
})), [])
hostname = string
lan_ip = string
lan_subnet = string
lan_gateway = string
install_disk = string
longhorn_disks = list(object({
device = string