path: root/src/com/android/tradefed/build/FileDownloadCache.java

/*
 * Copyright (C) 2011 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.android.tradefed.build;

import com.android.ddmlib.Log;
import com.android.tradefed.command.FatalHostError;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.util.FileUtil;

import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Stack;
import java.util.concurrent.locks.ReentrantLock;

/**
 * A helper class that maintains a local filesystem LRU cache of downloaded files.
 */
public class FileDownloadCache {

    private static final String LOG_TAG = "FileDownloadCache";

    private static final char REL_PATH_SEPARATOR = '/';

    /** fixed location of download cache. */
    private final File mCacheRoot;

    /**
     * The map of remote file paths to local files, stored in least-recently-used order.
     * <p/>
     * Used for performance reasons. Functionally speaking, this data structure is not needed,
     * since all info could be obtained from inspecting the filesystem.
     */
    private final Map<String, File> mCacheMap = new LinkedHashMap<String, File>();

    /** the lock for <var>mCacheMap</var> */
    private final ReentrantLock mCacheMapLock = new ReentrantLock();

    private long mCurrentCacheSize = 0;

    /** The approximate maximum allowed size of the local file cache. Default to 20 gig */
    private long mMaxFileCacheSize = 20L * 1024L * 1024L * 1024L;

    /**
     * Struct for a {@link File} and its remote relative path
     */
    private static class FilePair {
        final String mRelPath;
        final File mFile;

        FilePair(String relPath, File file) {
            mRelPath = relPath;
            mFile = file;
        }
    }

    /**
     * A {@link Comparator} for comparing {@link File}s based on {@link File#lastModified()}.
     */
    private static class FileTimeComparator implements Comparator<FilePair> {
        @Override
        public int compare(FilePair o1, FilePair o2) {
            Long timestamp1 = new Long(o1.mFile.lastModified());
            Long timestamp2 = o2.mFile.lastModified();
            return timestamp1.compareTo(timestamp2);
        }
    }

    /**
     * Create a {@link FileDownloadCache}, deleting any previous cache contents from disk.
     * <p/>
     * Assumes that the current process has exclusive access to the <var>cacheRoot</var> directory.
     * <p/>
     * Essentially, the LRU cache is a mirror of a given remote file path hierarchy.
     */
    FileDownloadCache(File cacheRoot) {
        mCacheRoot = cacheRoot;
        if (!mCacheRoot.exists()) {
            Log.d(LOG_TAG, String.format("Creating file cache at %s",
                    mCacheRoot.getAbsolutePath()));
            if (!mCacheRoot.mkdirs()) {
                throw new FatalHostError(String.format("Could not create cache directory at %s",
                        mCacheRoot.getAbsolutePath()));
            }
        } else {
            Log.d(LOG_TAG, String.format("Building file cache from contents at %s",
                    mCacheRoot.getAbsolutePath()));
            // create an unsorted list of all the files in mCacheRoot. Need to create list first
            // rather than inserting in Map directly because Maps cannot be sorted
            List<FilePair> cacheEntryList = new LinkedList<FilePair>();
            addFiles(mCacheRoot, new Stack<String>(), cacheEntryList);
            // now sort them based on file timestamp, to get them in LRU order
            Collections.sort(cacheEntryList, new FileTimeComparator());
            // now insert them into the map
            for (FilePair cacheEntry : cacheEntryList) {
                mCacheMap.put(cacheEntry.mRelPath, cacheEntry.mFile);
                mCurrentCacheSize += cacheEntry.mFile.length();
            }
            // this would be an unusual situation, but check if current cache is already too big
            if (mCurrentCacheSize > getMaxFileCacheSize()) {
                incrementAndAdjustCache(0);
            }
        }
    }

    /**
     * Recursive method for adding a directory's contents to the cache map
     * <p/>
     * cacheEntryList will contain results of all files found in cache, in no guaranteed order.
     *
     * @param dir the parent directory to search
     * @param relPathSegments the current filesystem path of <var>dir</var>, relative to
     *            <var>mCacheRoot</var>
     * @param cacheEntryList the list of files discovered
     */
    private void addFiles(File dir, Stack<String> relPathSegments,
            List<FilePair> cacheEntryList) {

        File[] fileList = dir.listFiles();
        if (fileList == null) {
            CLog.e("Unable to list files in cache dir %s", dir.getAbsolutePath());
            return;
        }
        for (File childFile : fileList) {
            if (childFile.isDirectory()) {
                relPathSegments.push(childFile.getName());
                addFiles(childFile, relPathSegments, cacheEntryList);
                relPathSegments.pop();
            } else if (childFile.isFile()) {
                StringBuffer relPath = new StringBuffer();
                for (String pathSeg : relPathSegments) {
                    relPath.append(pathSeg);
                    relPath.append(REL_PATH_SEPARATOR);
                }
                relPath.append(childFile.getName());
                cacheEntryList.add(new FilePair(relPath.toString(), childFile));
            } else {
                Log.w(LOG_TAG, String.format("Unrecognized file type %s in cache",
                        childFile.getAbsolutePath()));
            }
        }
    }

    /**
     * Set the maximum size of the local file cache.
     * <p/>
     * Cache will not be adjusted immediately if set to a smaller size than current, but will
     * take effect on next file download.
     * @param numBytes
     */
    public void setMaxCacheSize(long numBytes) {
        // for simplicity, get global lock
        mCacheMapLock.lock();
        mMaxFileCacheSize = numBytes;
        mCacheMapLock.unlock();
    }

    /**
     * Returns a local file corresponding to the given <var>remotePath</var>
     * <p/>
     * The local {@link File} will be copied from the cache if it exists, otherwise will be
     * downloaded via the given {@link IFileDownloader}.
     *
     * @param downloader the {@link IFileDownloader}
     * @param remotePath the remote file.
     * @return a local {@link File} containing contents of remotePath
     * @throws BuildRetrievalError if file could not be retrieved
     */
    public File fetchRemoteFile(IFileDownloader downloader, String remotePath)
            throws BuildRetrievalError {
        boolean download = false;
        // remove and then add previous cache entry to maintain LRU order
        mCacheMapLock.lock();
        File cachedFile = mCacheMap.remove(remotePath);
        if (cachedFile == null) {
            // create a local File that maps to remotePath
            // convert remotePath to a local path if necessary
            String localRelativePath = convertPath(remotePath);
            cachedFile = new File(mCacheRoot, localRelativePath);
            cachedFile.getParentFile().mkdirs();
            download = true;
        }
        // lock on the file, so no other thread attempts to delete it or access it before its
        // downloaded
        File copyFile = null;
        try {
            synchronized (cachedFile) {
                mCacheMap.put(remotePath, cachedFile);
                mCacheMapLock.unlock();
                if (download) {
                    downloadFile(downloader, remotePath, cachedFile);
                } else {
                    Log.d(LOG_TAG, String.format("Retrieved remote file %s from cached file %s",
                            remotePath, cachedFile.getAbsolutePath()));
                }
                copyFile = copyFile(remotePath, cachedFile);

            }
        } catch (BuildRetrievalError e) {
            // remove entry from cache outside of cachedFile lock, to prevent deadlock
            mCacheMapLock.lock();
            mCacheMap.remove(remotePath);
            mCacheMapLock.unlock();
            throw e;
        }
        if (download) {
           incrementAndAdjustCache(cachedFile.length());
        }
        return copyFile;
    }

    private void downloadFile(IFileDownloader downloader, String remotePath, File cachedFile)
            throws BuildRetrievalError {
        try {
            Log.d(LOG_TAG, String.format("Downloading %s to cache", remotePath));
            downloader.downloadFile(remotePath, cachedFile);
        } catch (BuildRetrievalError e) {
            // cached file is likely incomplete, delete it
            cachedFile.delete();
            throw e;
        }
    }

    private File copyFile(String remotePath, File cachedFile) throws BuildRetrievalError {
        // attempt to create a local copy of cached file with sane name
        File hardlinkFile = null;
        try {
            hardlinkFile = FileUtil.createTempFileForRemote(remotePath, null);
            hardlinkFile.delete();
            CLog.d("Creating hardlink '%s' to '%s'", hardlinkFile.getAbsolutePath(),
                    cachedFile.getAbsolutePath());
            FileUtil.hardlinkFile(cachedFile, hardlinkFile);
            return hardlinkFile;
        } catch (IOException e) {
            if (hardlinkFile != null) {
                hardlinkFile.delete();
            }
            // cached file might be corrupt or incomplete, delete it
            cachedFile.delete();
            throw new BuildRetrievalError(String.format("Failed to copy cached file %s",
                    cachedFile), e);
        }
    }

    /**
     * Convert remote relative path into an equivalent local path
     * @param remotePath
     * @return the local relative path
     */
    private String convertPath(String remotePath) {
        if (FileDownloadCache.REL_PATH_SEPARATOR != File.separatorChar) {
            return remotePath.replace(FileDownloadCache.REL_PATH_SEPARATOR , File.separatorChar);
        } else {
            // no conversion necessary
            return remotePath;
        }
    }

    /**
     * Adjust file cache size to mMaxFileCacheSize if necessary by deleting old files
     */
    private void incrementAndAdjustCache(long length) {
        mCacheMapLock.lock();
        try {
            mCurrentCacheSize += length;
            Iterator<Map.Entry<String, File>> mapIterator = mCacheMap.entrySet().iterator();
            // map cannot be modified while iterating, so store entries to be deleted in another list
            Collection<String> keysToDelete = new LinkedList<String>();
            while (mCurrentCacheSize > getMaxFileCacheSize() && mapIterator.hasNext()) {
                Map.Entry<String, File> currentEntry = mapIterator.next();
                keysToDelete.add(currentEntry.getKey());
                mCurrentCacheSize -= currentEntry.getValue().length();
            }
            for (String deleteKey : keysToDelete) {
                File deleteFile = mCacheMap.remove(deleteKey);
                synchronized (deleteFile) {
                    deleteFile.delete();
                }
            }
            // audit cache size
            if (mCurrentCacheSize < 0) {
                // should never happen
                Log.e(LOG_TAG, "Cache size is less than 0!");
                // TODO: throw fatal error?
            }
        } finally {
            mCacheMapLock.unlock();
        }
    }

    /**
     * Returns the cached file for given remote path, or <code>null</code> if no cached file exists.
     * <p/>
     * Exposed for unit testing
     *
     * @param remoteFilePath the remote file path
     * @return the cached {@link File} or <code>null</code>
     */
     File getCachedFile(String remoteFilePath) {
        mCacheMapLock.lock();
        try {
            return mCacheMap.get(remoteFilePath);
        } finally {
            mCacheMapLock.unlock();
        }
     }

    /**
     * Empty the cache, deleting all files.
     * <p/>
     * exposed for unit testing
     */
     void empty() {
        long currentMax = getMaxFileCacheSize();
        // reuse incrementAndAdjustCache to clear cache, by setting cache cap to 0
        setMaxCacheSize(0L);
        incrementAndAdjustCache(0);
        setMaxCacheSize(currentMax);
    }

    /**
     * Retrieve the oldest remotePath from cache.
     * <p/>
     * Exposed for unit testing
     *
     * @return the remote path or <code>null</null> if cache is empty
     */
    String getOldestEntry() {
        mCacheMapLock.lock();
        try {
            if (!mCacheMap.isEmpty()) {
                return mCacheMap.keySet().iterator().next();
            } else {
                return null;
            }
        } finally {
            mCacheMapLock.unlock();
        }
    }

    /**
     * Get the current max size of file cache.
     * <p/>
     * exposed for unit testing.
     *
     * @return the mMaxFileCacheSize
     */
    long getMaxFileCacheSize() {
        return mMaxFileCacheSize;
    }
}