1   package org.apache.lucene.store;
2   
3   /**
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements.  See the NOTICE file distributed with
6    * this work for additional information regarding copyright ownership.
7    * The ASF licenses this file to You under the Apache License, Version 2.0
8    * (the "License"); you may not use this file except in compliance with
9    * the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  import java.io.File;
21  import java.io.FileInputStream;
22  import java.io.FileOutputStream;
23  import java.io.IOException;
24  import java.io.RandomAccessFile;
25  import java.security.MessageDigest;
26  import java.security.NoSuchAlgorithmException;
27  import java.util.HashMap;
28  import java.util.Map;
29  
30  import org.apache.lucene.index.IndexFileNameFilter;
31  
32  // Used only for WRITE_LOCK_NAME in deprecated create=true case:
33  import org.apache.lucene.index.IndexWriter;
34  
35  /**
36   * Straightforward implementation of {@link Directory} as a directory of files.
37   * Locking implementation is by default the {@link SimpleFSLockFactory}, but
38   * can be changed either by passing in a {@link LockFactory} instance to
39   * <code>getDirectory</code>, or specifying the LockFactory class by setting
40   * <code>org.apache.lucene.store.FSDirectoryLockFactoryClass</code> Java system
41   * property, or by calling {@link #setLockFactory} after creating
42   * the Directory.
43  
44   * <p>Directories are cached, so that, for a given canonical
45   * path, the same FSDirectory instance will always be
46   * returned by <code>getDirectory</code>.  This permits
47   * synchronization on directories.</p>
48   *
49   * @see Directory
50   */
51  public class FSDirectory extends Directory {
52      
53    /** This cache of directories ensures that there is a unique Directory
54     * instance per path, so that synchronization on the Directory can be used to
55     * synchronize access between readers and writers.  We use
56     * refcounts to ensure when the last use of an FSDirectory
57     * instance for a given canonical path is closed, we remove the
58     * instance from the cache.  See LUCENE-776
59     * for some relevant discussion.
60     */
61    private static final Map DIRECTORIES = new HashMap();
62  
63    private static boolean disableLocks = false;
64  
65    // TODO: should this move up to the Directory base class?  Also: should we
66    // make a per-instance (in addition to the static "default") version?
67  
68    /**
69     * Set whether Lucene's use of lock files is disabled. By default, 
70     * lock files are enabled. They should only be disabled if the index
71     * is on a read-only medium like a CD-ROM.
72     */
73    public static void setDisableLocks(boolean doDisableLocks) {
74      FSDirectory.disableLocks = doDisableLocks;
75    }
76  
77    /**
78     * Returns whether Lucene's use of lock files is disabled.
79     * @return true if locks are disabled, false if locks are enabled.
80     */
81    public static boolean getDisableLocks() {
82      return FSDirectory.disableLocks;
83    }
84  
85    /**
86     * Directory specified by <code>org.apache.lucene.lockDir</code>
87     * or <code>java.io.tmpdir</code> system property.
88  
89     * @deprecated As of 2.1, <code>LOCK_DIR</code> is unused
90     * because the write.lock is now stored by default in the
91     * index directory.  If you really want to store locks
92     * elsewhere you can create your own {@link
93     * SimpleFSLockFactory} (or {@link NativeFSLockFactory},
94     * etc.) passing in your preferred lock directory.  Then,
95     * pass this <code>LockFactory</code> instance to one of
96     * the <code>getDirectory</code> methods that take a
97     * <code>lockFactory</code> (for example, {@link #getDirectory(String, LockFactory)}).
98     */
99    public static final String LOCK_DIR = System.getProperty("org.apache.lucene.lockDir",
100                                                            System.getProperty("java.io.tmpdir"));
101 
102   /** The default class which implements filesystem-based directories. */
103   private static Class IMPL;
104   static {
105     try {
106       String name =
107         System.getProperty("org.apache.lucene.FSDirectory.class",
108                            FSDirectory.class.getName());
109       IMPL = Class.forName(name);
110     } catch (ClassNotFoundException e) {
111       throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e);
112     } catch (SecurityException se) {
113       try {
114         IMPL = Class.forName(FSDirectory.class.getName());
115       } catch (ClassNotFoundException e) {
116         throw new RuntimeException("cannot load default FSDirectory class: " + e.toString(), e);
117       }
118     }
119   }
120 
121   private static MessageDigest DIGESTER;
122 
123   static {
124     try {
125       DIGESTER = MessageDigest.getInstance("MD5");
126     } catch (NoSuchAlgorithmException e) {
127         throw new RuntimeException(e.toString(), e);
128     }
129   }
130 
131   /** A buffer optionally used in renameTo method */
132   private byte[] buffer = null;
133 
134   /** Returns the directory instance for the named location.
135    * @param path the path to the directory.
136    * @return the FSDirectory for the named file.  */
137   public static FSDirectory getDirectory(String path)
138       throws IOException {
139     return getDirectory(new File(path), null);
140   }
141 
142   /** Returns the directory instance for the named location.
143    * @param path the path to the directory.
144    * @param lockFactory instance of {@link LockFactory} providing the
145    *        locking implementation.
146    * @return the FSDirectory for the named file.  */
147   public static FSDirectory getDirectory(String path, LockFactory lockFactory)
148       throws IOException {
149     return getDirectory(new File(path), lockFactory);
150   }
151 
152   /** Returns the directory instance for the named location.
153    * @param file the path to the directory.
154    * @return the FSDirectory for the named file.  */
155   public static FSDirectory getDirectory(File file)
156     throws IOException {
157     return getDirectory(file, null);
158   }
159 
160   /** Returns the directory instance for the named location.
161    * @param file the path to the directory.
162    * @param lockFactory instance of {@link LockFactory} providing the
163    *        locking implementation.
164    * @return the FSDirectory for the named file.  */
165   public static FSDirectory getDirectory(File file, LockFactory lockFactory)
166     throws IOException
167   {
168     file = new File(file.getCanonicalPath());
169 
170     if (file.exists() && !file.isDirectory())
171       throw new IOException(file + " not a directory");
172 
173     if (!file.exists())
174       if (!file.mkdirs())
175         throw new IOException("Cannot create directory: " + file);
176 
177     FSDirectory dir;
178     synchronized (DIRECTORIES) {
179       dir = (FSDirectory)DIRECTORIES.get(file);
180       if (dir == null) {
181         try {
182           dir = (FSDirectory)IMPL.newInstance();
183         } catch (Exception e) {
184           throw new RuntimeException("cannot load FSDirectory class: " + e.toString(), e);
185         }
186         dir.init(file, lockFactory);
187         DIRECTORIES.put(file, dir);
188       } else {
189         // Catch the case where a Directory is pulled from the cache, but has a
190         // different LockFactory instance.
191         if (lockFactory != null && lockFactory != dir.getLockFactory()) {
192           throw new IOException("Directory was previously created with a different LockFactory instance; please pass null as the lockFactory instance and use setLockFactory to change it");
193         }
194       }
195     }
196     synchronized (dir) {
197       dir.refCount++;
198     }
199     return dir;
200   }
201 
202 
203   /** Returns the directory instance for the named location.
204    *
205    * @deprecated Use IndexWriter's create flag, instead, to
206    * create a new index.
207    *
208    * @param path the path to the directory.
209    * @param create if true, create, or erase any existing contents.
210    * @return the FSDirectory for the named file.  */
211   public static FSDirectory getDirectory(String path, boolean create)
212       throws IOException {
213     return getDirectory(new File(path), create);
214   }
215 
216   /** Returns the directory instance for the named location.
217    *
218    * @deprecated Use IndexWriter's create flag, instead, to
219    * create a new index.
220    *
221    * @param file the path to the directory.
222    * @param create if true, create, or erase any existing contents.
223    * @return the FSDirectory for the named file.  */
224   public static FSDirectory getDirectory(File file, boolean create)
225     throws IOException
226   {
227     FSDirectory dir = getDirectory(file, null);
228 
229     // This is now deprecated (creation should only be done
230     // by IndexWriter):
231     if (create) {
232       dir.create();
233     }
234 
235     return dir;
236   }
237 
238   private void create() throws IOException {
239     if (directory.exists()) {
240       String[] files = directory.list(IndexFileNameFilter.getFilter());            // clear old files
241       if (files == null)
242         throw new IOException("cannot read directory " + directory.getAbsolutePath() + ": list() returned null");
243       for (int i = 0; i < files.length; i++) {
244         File file = new File(directory, files[i]);
245         if (!file.delete())
246           throw new IOException("Cannot delete " + file);
247       }
248     }
249     lockFactory.clearLock(IndexWriter.WRITE_LOCK_NAME);
250   }
251 
252   private File directory = null;
253   private int refCount;
254 
255   protected FSDirectory() {};                     // permit subclassing
256 
257   private void init(File path, LockFactory lockFactory) throws IOException {
258 
259     // Set up lockFactory with cascaded defaults: if an instance was passed in,
260     // use that; else if locks are disabled, use NoLockFactory; else if the
261     // system property org.apache.lucene.store.FSDirectoryLockFactoryClass is set,
262     // instantiate that; else, use SimpleFSLockFactory:
263 
264     directory = path;
265 
266     boolean doClearLockID = false;
267 
268     if (lockFactory == null) {
269 
270       if (disableLocks) {
271         // Locks are disabled:
272         lockFactory = NoLockFactory.getNoLockFactory();
273       } else {
274         String lockClassName = System.getProperty("org.apache.lucene.store.FSDirectoryLockFactoryClass");
275 
276         if (lockClassName != null && !lockClassName.equals("")) {
277           Class c;
278 
279           try {
280             c = Class.forName(lockClassName);
281           } catch (ClassNotFoundException e) {
282             throw new IOException("unable to find LockClass " + lockClassName);
283           }
284 
285           try {
286             lockFactory = (LockFactory) c.newInstance();          
287           } catch (IllegalAccessException e) {
288             throw new IOException("IllegalAccessException when instantiating LockClass " + lockClassName);
289           } catch (InstantiationException e) {
290             throw new IOException("InstantiationException when instantiating LockClass " + lockClassName);
291           } catch (ClassCastException e) {
292             throw new IOException("unable to cast LockClass " + lockClassName + " instance to a LockFactory");
293           }
294 
295           if (lockFactory instanceof NativeFSLockFactory) {
296             ((NativeFSLockFactory) lockFactory).setLockDir(path);
297           } else if (lockFactory instanceof SimpleFSLockFactory) {
298             ((SimpleFSLockFactory) lockFactory).setLockDir(path);
299           }
300         } else {
301           // Our default lock is SimpleFSLockFactory;
302           // default lockDir is our index directory:
303           lockFactory = new SimpleFSLockFactory(path);
304           doClearLockID = true;
305         }
306       }
307     }
308 
309     setLockFactory(lockFactory);
310 
311     if (doClearLockID) {
312       // Clear the prefix because write.lock will be
313       // stored in our directory:
314       lockFactory.setLockPrefix(null);
315     }
316   }
317 
318   /** Returns an array of strings, one for each Lucene index file in the directory. */
319   public String[] list() {
320     ensureOpen();
321     return directory.list(IndexFileNameFilter.getFilter());
322   }
323 
324   /** Returns true iff a file with the given name exists. */
325   public boolean fileExists(String name) {
326     ensureOpen();
327     File file = new File(directory, name);
328     return file.exists();
329   }
330 
331   /** Returns the time the named file was last modified. */
332   public long fileModified(String name) {
333     ensureOpen();
334     File file = new File(directory, name);
335     return file.lastModified();
336   }
337 
338   /** Returns the time the named file was last modified. */
339   public static long fileModified(File directory, String name) {
340     File file = new File(directory, name);
341     return file.lastModified();
342   }
343 
344   /** Set the modified time of an existing file to now. */
345   public void touchFile(String name) {
346     ensureOpen();
347     File file = new File(directory, name);
348     file.setLastModified(System.currentTimeMillis());
349   }
350 
351   /** Returns the length in bytes of a file in the directory. */
352   public long fileLength(String name) {
353     ensureOpen();
354     File file = new File(directory, name);
355     return file.length();
356   }
357 
358   /** Removes an existing file in the directory. */
359   public void deleteFile(String name) throws IOException {
360     ensureOpen();
361     File file = new File(directory, name);
362     if (!file.delete())
363       throw new IOException("Cannot delete " + file);
364   }
365 
366   /** Renames an existing file in the directory. 
367    * Warning: This is not atomic.
368    * @deprecated 
369    */
370   public synchronized void renameFile(String from, String to)
371       throws IOException {
372     ensureOpen();
373     File old = new File(directory, from);
374     File nu = new File(directory, to);
375 
376     /* This is not atomic.  If the program crashes between the call to
377        delete() and the call to renameTo() then we're screwed, but I've
378        been unable to figure out how else to do this... */
379 
380     if (nu.exists())
381       if (!nu.delete())
382         throw new IOException("Cannot delete " + nu);
383 
384     // Rename the old file to the new one. Unfortunately, the renameTo()
385     // method does not work reliably under some JVMs.  Therefore, if the
386     // rename fails, we manually rename by copying the old file to the new one
387     if (!old.renameTo(nu)) {
388       java.io.InputStream in = null;
389       java.io.OutputStream out = null;
390       try {
391         in = new FileInputStream(old);
392         out = new FileOutputStream(nu);
393         // see if the buffer needs to be initialized. Initialization is
394         // only done on-demand since many VM's will never run into the renameTo
395         // bug and hence shouldn't waste 1K of mem for no reason.
396         if (buffer == null) {
397           buffer = new byte[1024];
398         }
399         int len;
400         while ((len = in.read(buffer)) >= 0) {
401           out.write(buffer, 0, len);
402         }
403 
404         // delete the old file.
405         old.delete();
406       }
407       catch (IOException ioe) {
408         IOException newExc = new IOException("Cannot rename " + old + " to " + nu);
409         newExc.initCause(ioe);
410         throw newExc;
411       }
412       finally {
413         try {
414           if (in != null) {
415             try {
416               in.close();
417             } catch (IOException e) {
418               throw new RuntimeException("Cannot close input stream: " + e.toString(), e);
419             }
420           }
421         } finally {
422           if (out != null) {
423             try {
424               out.close();
425             } catch (IOException e) {
426               throw new RuntimeException("Cannot close output stream: " + e.toString(), e);
427             }
428           }
429         }
430       }
431     }
432   }
433 
434   /** Creates a new, empty file in the directory with the given name.
435       Returns a stream writing this file. */
436   public IndexOutput createOutput(String name) throws IOException {
437     ensureOpen();
438     File file = new File(directory, name);
439     if (file.exists() && !file.delete())          // delete existing, if any
440       throw new IOException("Cannot overwrite: " + file);
441 
442     return new FSIndexOutput(file);
443   }
444 
445   public void sync(String name) throws IOException {
446     ensureOpen();
447     File fullFile = new File(directory, name);
448     boolean success = false;
449     int retryCount = 0;
450     IOException exc = null;
451     while(!success && retryCount < 5) {
452       retryCount++;
453       RandomAccessFile file = null;
454       try {
455         try {
456           file = new RandomAccessFile(fullFile, "rw");
457           file.getFD().sync();
458           success = true;
459         } finally {
460           if (file != null)
461             file.close();
462         }
463       } catch (IOException ioe) {
464         if (exc == null)
465           exc = ioe;
466         try {
467           // Pause 5 msec
468           Thread.sleep(5);
469         } catch (InterruptedException ie) {
470           Thread.currentThread().interrupt();
471         }
472       }
473     }
474     if (!success)
475       // Throw original exception
476       throw exc;
477   }
478 
479   // Inherit javadoc
480   public IndexInput openInput(String name) throws IOException {
481     ensureOpen();
482     return openInput(name, BufferedIndexInput.BUFFER_SIZE);
483   }
484 
485   // Inherit javadoc
486   public IndexInput openInput(String name, int bufferSize) throws IOException {
487     ensureOpen();
488     return new FSIndexInput(new File(directory, name), bufferSize);
489   }
490 
491   /**
492    * So we can do some byte-to-hexchar conversion below
493    */
494   private static final char[] HEX_DIGITS =
495   {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
496 
497   
498   public String getLockID() {
499     ensureOpen();
500     String dirName;                               // name to be hashed
501     try {
502       dirName = directory.getCanonicalPath();
503     } catch (IOException e) {
504       throw new RuntimeException(e.toString(), e);
505     }
506 
507     byte digest[];
508     synchronized (DIGESTER) {
509       digest = DIGESTER.digest(dirName.getBytes());
510     }
511     StringBuffer buf = new StringBuffer();
512     buf.append("lucene-");
513     for (int i = 0; i < digest.length; i++) {
514       int b = digest[i];
515       buf.append(HEX_DIGITS[(b >> 4) & 0xf]);
516       buf.append(HEX_DIGITS[b & 0xf]);
517     }
518 
519     return buf.toString();
520   }
521 
522   /** Closes the store to future operations. */
523   public synchronized void close() {
524     if (isOpen && --refCount <= 0) {
525       isOpen = false;
526       synchronized (DIRECTORIES) {
527         DIRECTORIES.remove(directory);
528       }
529     }
530   }
531 
532   public File getFile() {
533     ensureOpen();
534     return directory;
535   }
536 
537   /** For debug output. */
538   public String toString() {
539     return this.getClass().getName() + "@" + directory;
540   }
541 
542   protected static class FSIndexInput extends BufferedIndexInput {
543   
544     protected static class Descriptor extends RandomAccessFile {
545       // remember if the file is open, so that we don't try to close it
546       // more than once
547       protected volatile boolean isOpen;
548       long position;
549       final long length;
550       
551       public Descriptor(File file, String mode) throws IOException {
552         super(file, mode);
553         isOpen=true;
554         length=length();
555       }
556   
557       public void close() throws IOException {
558         if (isOpen) {
559           isOpen=false;
560           super.close();
561         }
562       }
563   
564       protected void finalize() throws Throwable {
565         try {
566           close();
567         } finally {
568           super.finalize();
569         }
570       }
571     }
572   
573     protected final Descriptor file;
574     boolean isClone;
575   
576     public FSIndexInput(File path) throws IOException {
577       this(path, BufferedIndexInput.BUFFER_SIZE);
578     }
579   
580     public FSIndexInput(File path, int bufferSize) throws IOException {
581       super(bufferSize);
582       file = new Descriptor(path, "r");
583     }
584   
585     /** IndexInput methods */
586     protected void readInternal(byte[] b, int offset, int len)
587          throws IOException {
588       synchronized (file) {
589         long position = getFilePointer();
590         if (position != file.position) {
591           file.seek(position);
592           file.position = position;
593         }
594         int total = 0;
595         do {
596           int i = file.read(b, offset+total, len-total);
597           if (i == -1)
598             throw new IOException("read past EOF");
599           file.position += i;
600           total += i;
601         } while (total < len);
602       }
603     }
604   
605     public void close() throws IOException {
606       // only close the file if this is not a clone
607       if (!isClone) file.close();
608     }
609   
610     protected void seekInternal(long position) {
611     }
612   
613     public long length() {
614       return file.length;
615     }
616   
617     public Object clone() {
618       FSIndexInput clone = (FSIndexInput)super.clone();
619       clone.isClone = true;
620       return clone;
621     }
622   
623     /** Method used for testing. Returns true if the underlying
624      *  file descriptor is valid.
625      */
626     boolean isFDValid() throws IOException {
627       return file.getFD().valid();
628     }
629   }
630 
631   protected static class FSIndexOutput extends BufferedIndexOutput {
632     RandomAccessFile file = null;
633   
634     // remember if the file is open, so that we don't try to close it
635     // more than once
636     private volatile boolean isOpen;
637 
638     public FSIndexOutput(File path) throws IOException {
639       file = new RandomAccessFile(path, "rw");
640       isOpen = true;
641     }
642   
643     /** output methods: */
644     public void flushBuffer(byte[] b, int offset, int size) throws IOException {
645       file.write(b, offset, size);
646     }
647     public void close() throws IOException {
648       // only close the file if it has not been closed yet
649       if (isOpen) {
650         boolean success = false;
651         try {
652           super.close();
653           success = true;
654         } finally {
655           isOpen = false;
656           if (!success) {
657             try {
658               file.close();
659             } catch (Throwable t) {
660               // Suppress so we don't mask original exception
661             }
662           } else
663             file.close();
664         }
665       }
666     }
667   
668     /** Random-access methods */
669     public void seek(long pos) throws IOException {
670       super.seek(pos);
671       file.seek(pos);
672     }
673     public long length() throws IOException {
674       return file.length();
675     }
676     public void setLength(long length) throws IOException {
677       file.setLength(length);
678     }
679   }
680 }
681