lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (SOLR-9269) Ability to create/delete/list snapshots for a solr core
Date Thu, 21 Jul 2016 17:33:20 GMT

    [ https://issues.apache.org/jira/browse/SOLR-9269?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15388084#comment-15388084
] 

ASF GitHub Bot commented on SOLR-9269:
--------------------------------------

Github user hgadre commented on a diff in the pull request:

    https://github.com/apache/lucene-solr/pull/52#discussion_r71750235
  
    --- Diff: solr/core/src/java/org/apache/solr/core/snapshots/SolrSnapshotMetaDataManager.java
---
    @@ -0,0 +1,419 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.solr.core.snapshots;
    +
    +import java.io.IOException;
    +import java.lang.invoke.MethodHandles;
    +import java.util.ArrayList;
    +import java.util.Collection;
    +import java.util.Collections;
    +import java.util.HashMap;
    +import java.util.HashSet;
    +import java.util.List;
    +import java.util.Locale;
    +import java.util.Map;
    +import java.util.Map.Entry;
    +import java.util.Optional;
    +import java.util.stream.Collectors;
    +
    +import org.apache.lucene.codecs.CodecUtil;
    +import org.apache.lucene.index.DirectoryReader;
    +import org.apache.lucene.index.IndexCommit;
    +import org.apache.lucene.index.IndexDeletionPolicy;
    +import org.apache.lucene.index.IndexWriterConfig.OpenMode;
    +import org.apache.lucene.store.Directory;
    +import org.apache.lucene.store.IOContext;
    +import org.apache.lucene.store.IndexInput;
    +import org.apache.lucene.store.IndexOutput;
    +import org.apache.lucene.util.IOUtils;
    +import org.apache.solr.common.SolrException;
    +import org.apache.solr.common.SolrException.ErrorCode;
    +import org.apache.solr.core.DirectoryFactory;
    +import org.apache.solr.core.IndexDeletionPolicyWrapper;
    +import org.apache.solr.core.SolrCore;
    +import org.apache.solr.core.DirectoryFactory.DirContext;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +import com.google.common.base.Preconditions;
    +
    +/**
    + * This class is responsible to manage the persistent snapshots meta-data for the Solr
indexes. The
    + * persistent snapshots are implemented by relying on Lucene {@linkplain IndexDeletionPolicy}
    + * abstraction to configure a specific {@linkplain IndexCommit} to be retained. The
    + * {@linkplain IndexDeletionPolicyWrapper} in Solr uses this class to create/delete the
Solr index
    + * snapshots.
    + */
    +public class SolrSnapshotMetaDataManager {
    +  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
    +  public static final String SNAPSHOT_METADATA_DIR = "snapshot_metadata";
    +
    +  /**
    +   * A class defining the meta-data for a specific snapshot.
    +   */
    +  public static class SnapshotMetaData {
    +    private String name;
    +    private String indexDirPath;
    +    private long generationNumber;
    +
    +    public SnapshotMetaData(String name, String indexDirPath, long generationNumber)
{
    +      super();
    +      this.name = name;
    +      this.indexDirPath = indexDirPath;
    +      this.generationNumber = generationNumber;
    +    }
    +
    +    public String getName() {
    +      return name;
    +    }
    +
    +    public String getIndexDirPath() {
    +      return indexDirPath;
    +    }
    +
    +    public long getGenerationNumber() {
    +      return generationNumber;
    +    }
    +
    +    @Override
    +    public String toString() {
    +      StringBuilder builder = new StringBuilder();
    +      builder.append("SnapshotMetaData[name=");
    +      builder.append(name);
    +      builder.append(", indexDirPath=");
    +      builder.append(indexDirPath);
    +      builder.append(", generation=");
    +      builder.append(generationNumber);
    +      builder.append("]");
    +      return builder.toString();
    +    }
    +  }
    +
    +  /** Prefix used for the save file. */
    +  public static final String SNAPSHOTS_PREFIX = "snapshots_";
    +  private static final int VERSION_START = 0;
    +  private static final int VERSION_CURRENT = VERSION_START;
    +  private static final String CODEC_NAME = "solr-snapshots";
    +
    +  // The index writer which maintains the snapshots metadata
    +  private long nextWriteGen;
    +
    +  private final Directory dir;
    +
    +  /** Used to map snapshot name to snapshot meta-data. */
    +  protected final Map<String,SnapshotMetaData> nameToDetailsMapping = new HashMap<>();
    +  /** Used to figure out the *current* index data directory path */
    +  private final SolrCore solrCore;
    +
    +  /**
    +   * A constructor.
    +   *
    +   * @param dir The directory where the snapshot meta-data should be stored. Enables
updating
    +   *            the existing meta-data.
    +   * @throws IOException in case of errors.
    +   */
    +  public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir) throws IOException
{
    +    this(solrCore, dir, OpenMode.CREATE_OR_APPEND);
    +  }
    +
    +  /**
    +   * A constructor.
    +   *
    +   * @param dir The directory where the snapshot meta-data is stored.
    +   * @param mode CREATE If previous meta-data should be erased.
    +   *             APPEND If previous meta-data should be read and updated.
    +   *             CREATE_OR_APPEND Creates a new meta-data structure if one does not exist
    +   *                              Updates the existing structure if one exists.
    +   * @throws IOException in case of errors.
    +   */
    +  public SolrSnapshotMetaDataManager(SolrCore solrCore, Directory dir, OpenMode mode)
throws IOException {
    +    this.solrCore = solrCore;
    +    this.dir = dir;
    +
    +    if (mode == OpenMode.CREATE) {
    +      deleteSnapshotMetadataFiles();
    +    }
    +
    +    loadFromSnapshotMetadataFile();
    +
    +    if (mode == OpenMode.APPEND && nextWriteGen == 0) {
    +      throw new IllegalStateException("no snapshots stored in this directory");
    +    }
    +  }
    +
    +  /**
    +   * @return The snapshot meta-data directory
    +   */
    +  public Directory getSnapshotsDir() {
    +    return dir;
    +  }
    +
    +  /**
    +   * This method creates a new snapshot meta-data entry.
    +   *
    +   * @param name The name of the snapshot.
    +   * @param indexDirPath The directory path where the index files are stored.
    +   * @param gen The generation number for the {@linkplain IndexCommit} being snapshotted.
    +   * @throws IOException in case of I/O errors.
    +   */
    +  public synchronized void snapshot(String name, String indexDirPath, long gen) throws
IOException {
    +    Preconditions.checkNotNull(name);
    +
    +    log.info("Creating the snapshot named {} for core {} associated with index commit
with generation {} in directory {}"
    +        , name, solrCore.getName(), gen, indexDirPath);
    +
    +    if(nameToDetailsMapping.containsKey(name)) {
    +      throw new SolrException(ErrorCode.BAD_REQUEST, "A snapshot with name " + name +
" already exists");
    +    }
    +
    +    SnapshotMetaData d = new SnapshotMetaData(name, indexDirPath, gen);
    +    nameToDetailsMapping.put(name, d);
    +
    +    boolean success = false;
    +    try {
    +      persist();
    +      success = true;
    +    } finally {
    +      if (!success) {
    +        try {
    +          release(name);
    +        } catch (Exception e) {
    +          // Suppress so we keep throwing original exception
    +        }
    +      }
    +    }
    +  }
    +
    +  /**
    +   * This method deletes a previously created snapshot (if any).
    +   *
    +   * @param name The name of the snapshot to be deleted.
    +   * @return The snapshot meta-data if the snapshot with the snapshot name exists.
    +   * @throws IOException in case of I/O error
    +   */
    +  public synchronized Optional<SnapshotMetaData> release(String name) throws IOException
{
    +    log.info("Deleting the snapshot named {} for core {}", name, solrCore.getName());
    +    SnapshotMetaData result = nameToDetailsMapping.remove(Preconditions.checkNotNull(name));
    +    if(result != null) {
    +      boolean success = false;
    +      try {
    +        persist();
    +        success = true;
    +      } finally {
    +        if (!success) {
    +          nameToDetailsMapping.put(name, result);
    +        }
    +      }
    +    }
    +    return Optional.ofNullable(result);
    +  }
    +
    +  /**
    +   * This method returns if snapshot is created for the specified generation number in
    +   * the *current* index directory.
    +   *
    +   * @param genNumber The generation number for the {@linkplain IndexCommit} to be checked.
    +   * @return true if the snapshot is created.
    +   *         false otherwise.
    +   */
    +  public synchronized boolean isSnapshotted(long genNumber) {
    +    return isSnapshotted(solrCore.getIndexDir(), genNumber);
    +  }
    +
    +  /**
    +   * This method returns if snapshot is created for the specified generation number in
    +   * the specified index directory.
    +   *
    +   * @param genNumber The generation number for the {@linkplain IndexCommit} to be checked.
    +   * @return true if the snapshot is created.
    +   *         false otherwise.
    +   */
    +  public synchronized boolean isSnapshotted(String indexDirPath, long genNumber) {
    +    return nameToDetailsMapping.values().stream()
    --- End diff --
    
    Done.


> Ability to create/delete/list snapshots for a solr core
> -------------------------------------------------------
>
>                 Key: SOLR-9269
>                 URL: https://issues.apache.org/jira/browse/SOLR-9269
>             Project: Solr
>          Issue Type: Sub-task
>          Components: SolrCloud
>            Reporter: Hrishikesh Gadre
>            Assignee: David Smiley
>         Attachments: SOLR-9269.patch
>
>
> Support snapshot create/delete/list functionality @ the Solr core level. Please refer
to parent JIRA for more details.



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Mime
View raw message