1. Problem phenomenon
When I was doing testing today, I couldn’t start the pod in a namespace. When I checked the ns status, it kept showing Terminating.
[root@node1 ~]# kubectl get ns NAME STATUS AGE configmap Terminating 135d defaultActive 207d Harbor Active 207d kube-flannel Terminating 17m kube-node-lease Active 207d kube-public Active 207d kube-system Active 207d kubekey-system Active 207d kubernetes-dashboard Active 207d local-path-storage Active 187d nginx Active 146d test terminating 126d
Using –force to delete is also stuck.
[root@node1 ~]# kubectl delete ns test --force Warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely. namespace "test" force deleted
2. View resources under ns
Based on the above phenomenon, it is suspected that there are unreleased resources under the ns. Use the following command to check
1: View all resources [root@node1 ~]# kubectl get all -n test No resources found in test namespace. [root@node1 ~]# It is learned from the above that there are no unreleased resources under this ns. ################## I saw on the Internet that you can use the command kubectl api-resources -o name --verbs=list --namespaced | Is empty
3. Problem handling
Export ns details in json format
[root@node1 ~]# kubectl get ns test -o json > test.json
Edit the test.json file and ensure that the content in the spec is empty, as follows:
"spec": {
"finalizers": [ #########
"kubernetes" ######### Delete these three lines and tell k8s that the content in the ns to be deleted is empty.
] #########
},
The empty ns overwrites the original ns by calling the k8s api interface.
[root@node1 ~]# curl -k \ > -H "Content-Type: application/json" \ > -X PUT \ > --data-binary @test.json \ > http://127.0.0.1:8081/api/v1/namespaces/test/finalize curl: (7) Failed connect to 127.0.0.1:8081; Connection refused Because the k8s master node uses authentication, if you use the command directly, the connection will be refused. You need to use kube-proxy to proxy port 8081.
Use kube-proxy to open the port
[root@node1 ~]# kubectl proxy --port=8081 Starting to serve on 127.0.0.1:8081
Open a new terminal and execute the above command again to return the following content
[root@node1 ~]# curl -k -H "Content-Type: application/json" -X PUT --data-binary @test.json http://127.0.0.1:8081/api/v1/ namespaces/test/finalize
{
"kind": "Namespace",
"apiVersion": "v1",
"metadata": {
"name": "test",
"uid": "f2676c45-a75b-49be-9e01-84958bedc4a0",
"resourceVersion": "33942216",
"creationTimestamp": "2023-06-28T06:02:07Z",
"deletionTimestamp": "2023-06-28T06:02:24Z",
"labels": {
"kubernetes.io/metadata.name": "test"
},
"annotations": {
"kubectl.kubernetes.io/last-applied-configuration": "{"apiVersion":"v1","kind":"Namespace","metadata":{"annotations ":{},"creationTimestamp":"2023-06-28T06:02:07Z","deletionTimestamp":"2023-06-28T06:02:24Z","labels\ ":{"kubernetes.io/metadata.name":"test"},"name":"test","resourceVersion":"28081915","uid\ ":"f2676c45-a75b-49be-9e01-84958bedc4a0"},"spec":{},"status":{"conditions":[{"lastTransitionTime":"2023 -06-28T06:02:29Z","message":"Discovery failed for some groups, 1 failing: unable to retrieve the complete list of server APIs: metrics.k8s.io/v1beta1: an error on the server ("Internal Server Error: \"/apis/metrics.k8s.io/v1beta1\": the server could not find the requested resource") has prevented the request from succeeding","reason":"DiscoveryFailed","status":"True","type":"NamespaceDeletionDiscoveryFailure"},{"lastTransitionTime ":"2023-06-28T06:02:29Z","message":"All legacy kube types successfully parsed","reason":"ParsedGroupVersions","status\ ":"False","type":"NamespaceDeletionGroupVersionParsingFailure"},{"lastTransitionTime":"2023-06-28T06:02:29Z","message":" All content successfully deleted, may be waiting on finalization","reason":"ContentDeleted","status":"False","type":"NamespaceDeletionContentFailure"}, {"lastTransitionTime":"2023-06-28T06:02:29Z","message":"All content successfully removed","reason":"ContentRemoved"," status":"False","type":"NamespaceContentRemaining"},{"lastTransitionTime":"2023-06-28T06:02:29Z","message": "All content-preserving finalizers finished","reason":"ContentHasNoFinalizers","status":"False","type":"NamespaceFinalizersRemaining"}],\ "phase":"Terminating"}}\
"
},
"managedFields": [
{
"manager": "kubectl-create",
"operation": "Update",
"apiVersion": "v1",
"time": "2023-06-28T06:02:07Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:labels": {
".": {},
"f:kubernetes.io/metadata.name": {}
}
}
}
},
{
"manager": "kube-controller-manager",
"operation": "Update",
"apiVersion": "v1",
"time": "2023-09-27T07:02:23Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:status": {
"f:conditions": {
".": {},
"k:{"type":"NamespaceContentRemaining"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
},
"k:{"type":"NamespaceDeletionContentFailure"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
},
"k:{"type":"NamespaceDeletionDiscoveryFailure"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
},
"k:{"type":"NamespaceDeletionGroupVersionParsingFailure"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
},
"k:{"type":"NamespaceFinalizersRemaining"}": {
".": {},
"f:lastTransitionTime": {},
"f:message": {},
"f:reason": {},
"f:status": {},
"f:type": {}
}
}
}
},
"subresource": "status"
},
{
"manager": "kubectl-client-side-apply",
"operation": "Update",
"apiVersion": "v1",
"time": "2023-11-02T02:46:01Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:kubectl.kubernetes.io/last-applied-configuration": {}
}
}
}
}
]
},
"spec": {},
"status": {
"phase": "Terminating",
"conditions": [
{
"type": "NamespaceDeletionDiscoveryFailure",
"status": "True",
"lastTransitionTime": "2023-06-28T06:02:29Z",
"reason": "DiscoveryFailed",
"message": "Discovery failed for some groups, 1 failing: unable to retrieve the complete list of server APIs: metrics.k8s.io/v1beta1: an error on the server ("Internal Server Error: "/apis /metrics.k8s.io/v1beta1": the server could not find the requested resource") has prevented the request from succeeding"
},
{
"type": "NamespaceDeletionGroupVersionParsingFailure",
"status": "False",
"lastTransitionTime": "2023-06-28T06:02:29Z",
"reason": "ParsedGroupVersions",
"message": "All legacy kube types successfully parsed"
},
{
"type": "NamespaceDeletionContentFailure",
"status": "False",
"lastTransitionTime": "2023-06-28T06:02:29Z",
"reason": "ContentDeleted",
"message": "All content successfully deleted, may be waiting on finalization"
},
{
"type": "NamespaceContentRemaining",
"status": "False",
"lastTransitionTime": "2023-06-28T06:02:29Z",
"reason": "ContentRemoved",
"message": "All content successfully removed"
},
{
"type": "NamespaceFinalizersRemaining",
"status": "False",
"lastTransitionTime": "2023-06-28T06:02:29Z",
"reason": "ContentHasNoFinalizers",
"message": "All content-preserving finalizers finished"
}
]
}
}
Check if ns has been deleted
[root@node1 ~]#kubectl get ns ####The ns named test has been deleted NAME STATUS AGE configmap Terminating 135d defaultActive 207d Harbor Active 207d kube-flannel Terminating 21m kube-node-lease Active 207d kube-public Active 207d kube-system Active 207d kubekey-system Active 207d kubernetes-dashboard Active 207d local-path-storage Active 187d nginx Active 146d Other ns in the Terminating state can also be deleted using the above method. ####################### Note that the name of ns should not be the same as the resource type, otherwise the ns name will be recognized as a resource type and an error will be reported. This is the problem with the ns named configmap above. As a result, the deletion error cannot be cleared, and it can only be deleted by calling etcd.