Skip to content

Fix gc lock removals #6166

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ddanielr merged 3 commits into from
Mar 4, 2026
+29 −11
Merged

Fix gc lock removals #6166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
33cf936
Fix gc lock removals
ddanielr Mar 3, 2026
e35c97d
Improvements to Admin stop command
ddanielr Mar 4, 2026
31dc235
Remove duplicate exists check
ddanielr Mar 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions core/src/main/java/org/apache/accumulo/core/fate/zookeeper/ServiceLock.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -825,13 +825,20 @@ public static void deleteLock(ZooReaderWriter zk, ServiceLockPath path)
public static void deleteLock(ZooReaderWriter zoo, String path,
ServerServices.Service serviceType, Predicate<HostAndPort> hostPortPredicate,
Consumer<String> messageOutput, Boolean dryRun) throws KeeperException, InterruptedException {

Objects.requireNonNull(path, "Lock path cannot be null");
Objects.requireNonNull(hostPortPredicate, "host predicate cannot be null");

var lockData = ServiceLock.getLockData(zoo.getZooKeeper(), ServiceLock.path(path));
if (lockData != null) {
ServerServices lock = new ServerServices(new String(lockData, UTF_8));
if (hostPortPredicate.test(lock.getAddress(serviceType))) {
messageOutput.accept("Deleting " + path + " from zookeeper");
if (!dryRun) {
zoo.recursiveDelete(path, NodeMissingPolicy.SKIP);
List<String> children = zoo.getChildren(path);
for (String child : children) {
messageOutput.accept("Deleting " + path + "/" + child + " from zookeeper");
if (!dryRun) {
zoo.recursiveDelete(path + "/" + child, NodeMissingPolicy.SKIP);
}
}
}
}
Expand Down
17 changes: 13 additions & 4 deletions server/base/src/main/java/org/apache/accumulo/server/util/Admin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.AddressUtil;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.core.util.tables.TableMap;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.server.cli.ServerUtilOpts;
Expand Down Expand Up @@ -589,20 +590,28 @@ private static void stopServers(final ServerContext context, List<String> server
var iid = context.getInstanceID();

String tserversPath = Constants.ZROOT + "/" + iid + Constants.ZTSERVERS;
ZooZap.removeLocks(zk, tserversPath, hostAndPort::contains, opts);
ServiceLock.deleteLocks(zk, tserversPath, hostAndPort::contains, log::debug, false);
String compactorsBasepath = Constants.ZROOT + "/" + iid + Constants.ZCOMPACTORS;
ZooZap.removeCompactorGroupedLocks(zk, compactorsBasepath, rg -> true,
hostAndPort::contains, opts);
String sserversPath = Constants.ZROOT + "/" + iid + Constants.ZSSERVERS;
ZooZap.removeScanServerGroupLocks(zk, sserversPath, hostAndPort::contains, rg -> true,
opts);
try {
ZooZap.removeScanServerGroupLocks(zk, sserversPath, hostAndPort::contains, rg -> true,
opts);
} catch (IllegalStateException e) {
log.debug("No Scan Server locks currently exist", e);
}

String managerLockPath = Constants.ZROOT + "/" + iid + Constants.ZMANAGER_LOCK;
ZooZap.removeSingletonLock(zk, managerLockPath, hostAndPort::contains, opts);
String gcLockPath = Constants.ZROOT + "/" + iid + Constants.ZGC_LOCK;
ZooZap.removeSingletonLock(zk, gcLockPath, hostAndPort::contains, opts);
ServiceLock.deleteLock(zk, gcLockPath, ServerServices.Service.GC_CLIENT,
hostAndPort::contains, log::debug, opts.dryRun);
String monitorLockPath = Constants.ZROOT + "/" + iid + Constants.ZMONITOR_LOCK;
ZooZap.removeSingletonLock(zk, monitorLockPath, hostAndPort::contains, opts);
String compactionCoordinatorLockPath =
Constants.ZROOT + "/" + iid + Constants.ZCOORDINATOR_LOCK;
ZooZap.removeSingletonLock(zk, compactionCoordinatorLockPath, hostAndPort::contains, opts);
} else {
for (var server : hostAndPort) {
signalGracefulShutdown(context, server);
Expand Down
4 changes: 2 additions & 2 deletions test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,10 +118,10 @@ private void killMacGc() throws ProcessNotFoundException, InterruptedException,
getCluster().killProcess(ServerType.GARBAGE_COLLECTOR,
getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR).iterator().next());
// delete lock in zookeeper if there, this will allow next GC to start quickly
var path = ServiceLock.path(getServerContext().getZooKeeperRoot() + Constants.ZGC_LOCK);
String path = getServerContext().getZooKeeperRoot() + Constants.ZGC_LOCK;
ZooReaderWriter zk = getServerContext().getZooReaderWriter();
try {
ServiceLock.deleteLock(zk, path);
ServiceLock.deleteLock(zk, path, Service.GC_CLIENT, hostAndPort -> true, log::debug, false);
} catch (IllegalStateException e) {
log.error("Unable to delete ZooLock for mini accumulo-gc", e);
}
Expand Down
6 changes: 4 additions & 2 deletions test/src/main/java/org/apache/accumulo/test/upgrade/GCUpgrade9to10TestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.util.ServerServices;
import org.apache.accumulo.manager.upgrade.Upgrader9to10;
import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
Expand Down Expand Up @@ -84,10 +85,11 @@ private void killMacGc() throws ProcessNotFoundException, InterruptedException,
getCluster().killProcess(ServerType.GARBAGE_COLLECTOR,
getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR).iterator().next());
// delete lock in zookeeper if there, this will allow next GC to start quickly
var path = ServiceLock.path(getServerContext().getZooKeeperRoot() + Constants.ZGC_LOCK);
String path = getServerContext().getZooKeeperRoot() + Constants.ZGC_LOCK;
ZooReaderWriter zk = getServerContext().getZooReaderWriter();
try {
ServiceLock.deleteLock(zk, path);
ServiceLock.deleteLock(zk, path, ServerServices.Service.GC_CLIENT, hostAndPort -> true,
log::debug, false);
} catch (IllegalStateException e) {
log.error("Unable to delete ZooLock for mini accumulo-gc", e);
}
Expand Down