HIVE-29528: Managed database directory not deleted after DROP DATABASE when hive.acid.lockless.reads.enabled is true

Author: Venu Reddy

ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CleanupRequest.java

@@ -30,6 +30,8 @@ public class CleanupRequest {
   private final String location;
   private final List<Path> obsoleteDirs;
   private final boolean purge;
+  private final boolean softDelete;
+  private final boolean sourceOfReplication;
   private final String runAs;
   private final String dbName;
   private final String fullPartitionName;
@@ -38,6 +40,8 @@ public CleanupRequest(CleanupRequestBuilder builder) {
     this.location = builder.location;
     this.obsoleteDirs = builder.obsoleteDirs;
     this.purge = builder.purge;
+    this.softDelete = builder.isSoftDelete;
+    this.sourceOfReplication = builder.sourceOfReplication;
     this.runAs = builder.runAs;
     this.dbName = builder.dbName;
     this.fullPartitionName = builder.fullPartitionName;
@@ -55,6 +59,14 @@ public boolean isPurge() {
     return purge;
   }
 
+  public boolean isSoftDelete() {
+    return softDelete;
+  }
+
+  public boolean isSourceOfReplication() {
+    return sourceOfReplication;
+  }
+
   public String runAs() {
     return runAs;
   }
@@ -74,6 +86,8 @@ public static class CleanupRequestBuilder {
     private String location;
     private List<Path> obsoleteDirs;
     private boolean purge;
+    private boolean isSoftDelete;
+    private boolean sourceOfReplication;
     private String runAs;
     private String dbName;
     private String fullPartitionName;
@@ -93,6 +107,16 @@ public CleanupRequestBuilder setPurge(boolean purge) {
       return this;
     }
 
+    public CleanupRequestBuilder setSoftDelete(boolean softDelete) {
+      this.isSoftDelete = softDelete;
+      return this;
+    }
+
+    public CleanupRequestBuilder setSourceOfReplication(boolean sourceOfReplication) {
+      this.sourceOfReplication = sourceOfReplication;
+      return this;
+    }
+
     public CleanupRequestBuilder setDbName(String dbName) {
       this.dbName = dbName;
       return this;

ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/FSRemover.java

@@ -17,8 +17,10 @@
  */
 package org.apache.hadoop.hive.ql.txn.compactor;
 
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.AcidConstants;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.ReplChangeManager;
 import org.apache.hadoop.hive.metastore.api.Database;
@@ -99,17 +101,18 @@ private List<Path> removeFiles(CleanupRequest cr)
     LOG.info("About to remove {} obsolete directories from {}. {}", cr.getObsoleteDirs().size(),
             cr.getLocation(), CompactorUtil.getDebugInfo(cr.getObsoleteDirs()));
     boolean needCmRecycle;
+    Database db = null;
     try {
-      Database db = metadataCache.computeIfAbsent(cr.getDbName(),
+      db = metadataCache.computeIfAbsent(cr.getDbName(),
               () -> CompactorUtil.resolveDatabase(conf, cr.getDbName()));
       needCmRecycle = ReplChangeManager.isSourceOfReplication(db);
     } catch (NoSuchObjectException ex) {
       // can not drop a database which is a source of replication
-      needCmRecycle = false;
+      needCmRecycle = cr.isSourceOfReplication();
     } catch (RuntimeException ex) {
       if (ex.getCause() instanceof NoSuchObjectException) {
         // can not drop a database which is a source of replication
-        needCmRecycle = false;
+        needCmRecycle = cr.isSourceOfReplication();
       } else {
         throw ex;
       }
@@ -126,6 +129,25 @@ private List<Path> removeFiles(CleanupRequest cr)
         deleted.add(dead);
       }
     }
+    if (db == null && cr.isSoftDelete()) {
+      Path databasePath = new Path(cr.getLocation()).getParent();
+      if (canRemoveDatabaseDir(fs, databasePath)) {
+        if (needCmRecycle) {
+          replChangeManager.recycle(databasePath, ReplChangeManager.RecycleType.MOVE, cr.isPurge());
+        }
+        FileUtils.deleteDir(fs, databasePath, cr.isPurge(), conf);
+      }
+    }
     return deleted;
   }
+
+  private boolean canRemoveDatabaseDir(FileSystem fs, Path databasePath) throws IOException {
+    for (FileStatus status : fs.listStatus(databasePath)) {
+      if (status.isDirectory() &&
+          status.getPath().getName().matches("(.*)" + AcidConstants.SOFT_DELETE_TABLE_PATTERN)) {
+        return false;
+      }
+    }
+    return true;
+  }
 }

ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/CompactionCleaner.java

@@ -286,6 +286,8 @@ private CleanupRequest getCleaningRequestBasedOnLocation(CompactionInfo ci, Stri
         .setFullPartitionName(ci.getFullPartitionName())
         .setRunAs(ci.runAs)
         .setPurge(ifPurge)
+        .setSoftDelete(ci.isDatabaseSoftDelete())
+        .setSourceOfReplication(ci.isSourceOfReplication())
         .setObsoleteDirs(Collections.singletonList(obsoletePath))
         .build();
   }

standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java

@@ -19,6 +19,8 @@
 package org.apache.hadoop.hive.metastore;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.AcidConstants;
+import org.apache.hadoop.hive.common.repl.ReplConst;
 import org.apache.hadoop.hive.metastore.api.CompactionRequest;
 import org.apache.hadoop.hive.metastore.api.CompactionType;
 import org.apache.hadoop.hive.metastore.api.Database;
@@ -91,6 +93,14 @@ public void onDropTable(DropTableEvent tableEvent) throws MetaException {
             rqst.setRunas(TxnUtils.findUserToRunAs(table.getSd().getLocation(), table, conf));
             rqst.putToProperties("location", table.getSd().getLocation());
             rqst.putToProperties("ifPurge", Boolean.toString(isMustPurge(tableEvent.getEnvironmentContext(), table)));
+            if ("true".equalsIgnoreCase(tableEvent.getEnvironmentContext().getProperties().get(
+                hive_metastoreConstants.SOFT_DELETE_DATABASE))) {
+              rqst.putToProperties(hive_metastoreConstants.SOFT_DELETE_DATABASE, Boolean.TRUE.toString());
+              if ("true".equalsIgnoreCase(tableEvent.getEnvironmentContext().getProperties().get(
+                  ReplConst.SOURCE_OF_REPLICATION))) {
+                rqst.putToProperties(ReplConst.SOURCE_OF_REPLICATION, Boolean.TRUE.toString());
+              }
+            }
             txnHandler.submitForCleanup(rqst, table.getWriteId(), currentTxn);
           } catch (InterruptedException | IOException e) {
             throwMetaException(e);

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java

@@ -28,6 +28,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.repl.ReplConst;
 import org.apache.hadoop.hive.metastore.Batchable;
 import org.apache.hadoop.hive.metastore.HMSHandler;
 import org.apache.hadoop.hive.metastore.IHMSHandler;
@@ -136,6 +137,10 @@ public DropDatabaseResult execute() throws TException, IOException {
         if (isSoftDelete) {
           context = new EnvironmentContext();
           context.putToProperties(hive_metastoreConstants.TXN_ID, String.valueOf(request.getTxnId()));
+          context.putToProperties(hive_metastoreConstants.SOFT_DELETE_DATABASE, Boolean.TRUE.toString());
+          if (ReplChangeManager.isSourceOfReplication(db)) {
+            context.putToProperties(ReplConst.SOURCE_OF_REPLICATION, Boolean.TRUE.toString());
+          }
           request.setDeleteManagedDir(false);
         }
         DropTableRequest dropRequest = new DropTableRequest(name, table.getTableName());

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/handler/DropDatabaseHandler.java

@@ -19,8 +19,10 @@
 
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
+import org.apache.hadoop.hive.common.repl.ReplConst;
 import org.apache.hadoop.hive.metastore.api.CompactionInfoStruct;
 import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.OptionalCompactionInfoStruct;
 import org.apache.hadoop.hive.metastore.api.TableValidWriteIds;
@@ -368,4 +370,12 @@ public void setWriteIds(boolean hasUncompactedAborts, Set<Long> writeIds) {
   public boolean isAbortedTxnCleanup() {
     return type == CompactionType.ABORT_TXN_CLEANUP;
   }
+
+  public boolean isDatabaseSoftDelete() {
+    return "true".equalsIgnoreCase(getProperty(hive_metastoreConstants.SOFT_DELETE_DATABASE));
+  }
+
+  public boolean isSourceOfReplication() {
+    return "true".equalsIgnoreCase(getProperty(ReplConst.SOURCE_OF_REPLICATION));
+  }
 }