1 /*------------------------------------------------------------------------------
2 Name: DirectoryManager.java
3 Project: xmlBlaster.org
4 Copyright: xmlBlaster.org, see xmlBlaster-LICENSE file
5 ------------------------------------------------------------------------------*/
6
7 package org.xmlBlaster.client.filepoller;
8
9 import java.io.BufferedInputStream;
10 import java.io.File;
11 import java.io.FileFilter;
12 import java.io.FileInputStream;
13 import java.io.FileOutputStream;
14 import java.io.IOException;
15 import java.io.InputStream;
16 import java.util.HashMap;
17 import java.util.HashSet;
18 import java.util.Iterator;
19 import java.util.Map;
20 import java.util.Set;
21 import java.util.TreeSet;
22
23 import java.util.logging.Logger;
24 import java.util.logging.Level;
25 import org.xmlBlaster.util.Global;
26 import org.xmlBlaster.util.XmlBlasterException;
27 import org.xmlBlaster.util.def.ErrorCode;
28
29 /**
30 * DirectoryManager
31 * @author <a href="mailto:michele@laghi.eu">Michele Laghi</a>
32 * @deprectated it is now replaced by the corresponding class in org.xmlBlaster.contrib.filewatcher
33 */
34 public class DirectoryManager {
35 private String ME = "DirectoryManager";
36 private Global global;
37 private static Logger log = Logger.getLogger(DirectoryManager.class.getName());
38 private long delaySinceLastFileChange;
39
40 private String directoryName;
41 private File directory;
42 /** this is the the directory where files are moved to after successful publishing. If null they will be erased */
43 private File sentDirectory;
44 /** this is the name of the directory where files are moved if they could not be send (too big) */
45 private File discardedDirectory;
46
47 private Map directoryEntries;
48 /** all files matching the filter will be processed. Null means everything will be processed */
49 private FileFilter fileFilter;
50 /** if set, then files will only be published when the lock-file has been removed. */
51 private FileFilter lockExtention;
52 /** convenience for performance: if lockExtention is '*.gif', then this will be '.gif' */
53 private String lockExt;
54
55 private Set lockFiles;
56
57 private boolean copyOnMove;
58
59 public DirectoryManager(Global global, String name, String directoryName, long delaySinceLastFileChange, String filter, String sent, String discarded, String lockExtention, boolean trueRegex, boolean copyOnMove) throws XmlBlasterException {
60 ME += "-" + name;
61 this.global = global;
62 if (filter != null)
63 this.fileFilter = new FilenameFilter(filter, trueRegex);
64
65 this.delaySinceLastFileChange = delaySinceLastFileChange;
66 this.directoryEntries = new HashMap();
67 this.directoryName = directoryName;
68 this.directory = initDirectory(null, "directoryName", directoryName);
69 if (this.directory == null)
70 throw new XmlBlasterException(this.global, ErrorCode.INTERNAL_NULLPOINTER, ME + ".constructor", "the directory '" + directoryName + "' is null");
71 this.sentDirectory = initDirectory(this.directory, "sent", sent);
72 this.discardedDirectory = initDirectory(this.directory, "discarded", discarded);
73 if (lockExtention != null) {
74 String tmp = lockExtention.trim();
75 if (!tmp.startsWith("*.")) {
76 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_CONFIGURATION, ME, "lockExtention must start with '*.' and be of the kind '*.lck'");
77 }
78 this.lockExtention = new FilenameFilter(tmp, false);
79 this.lockExt = tmp.substring(1); // '*.gif' -> '.gif'
80 }
81 this.lockFiles = new HashSet();
82 this.copyOnMove = copyOnMove;
83 }
84
85 /**
86 * Returns the specified directory or null or if needed it will create one
87 * @param propName
88 * @param dirName
89 * @return
90 * @throws XmlBlasterException
91 */
92 private File initDirectory(File parent, String propNameForLogging, String dirName) throws XmlBlasterException {
93 File dir = null;
94 if (dirName != null) {
95 File tmp = new File(dirName);
96 if (tmp.isAbsolute() || parent == null) {
97 dir = new File(dirName);
98 }
99 else {
100 dir = new File(parent, dirName);
101 }
102 if (!dir.exists()) {
103 String absDirName = null;
104 try {
105 absDirName = dir.getCanonicalPath();
106 }
107 catch (IOException ex) {
108 absDirName = dir.getAbsolutePath();
109 }
110 log.info(ME+": Constructor: directory '" + absDirName + "' does not yet exist. I will create it");
111 boolean ret = dir.mkdir();
112 if (!ret)
113 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME, "could not create directory '" + absDirName + "'");
114 }
115 if (!dir.isDirectory()) {
116 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME, "'" + dir.getAbsolutePath() + "' is not a directory");
117 }
118 if (!dir.canRead())
119 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".constructor", "no rights to read from the directory '" + dir.getAbsolutePath() + "'");
120 if (!dir.canWrite())
121 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".constructor", "no rights to write to the directory '" + dir.getAbsolutePath() + "'");
122 }
123 else {
124 log.info(ME+": Constructor: the '" + propNameForLogging + "' property is not set. Instead of moving concerned entries they will be deleted");
125 }
126 return dir;
127 }
128
129 /**
130 * Retrieves all files from the specified directory
131 * @param directory
132 * @return never returns null.
133 * @throws XmlBlasterException
134 */
135 private Map getNewFiles(File directory) throws XmlBlasterException {
136 if (this.lockExtention != null) { // reset lockFile set
137 this.lockFiles.clear();
138 }
139
140 this.directory = initDirectory(null, "directoryName", this.directoryName);
141 if (!directory.canRead())
142 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".scan", "I don't have rights to read from '" + directory.getName() + "'");
143 if (!directory.canWrite())
144 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".scan", "I don't have rights to write to '" + directory.getName() + "'");
145 File[] files = directory.listFiles(this.fileFilter);
146 if (files == null || files.length == 0)
147 return new HashMap();
148 if (this.lockExtention != null) {
149 // and then retrieve all lock files (this must be done after having got 'files' to avoid any gaps
150 File[] lckFiles = directory.listFiles(this.lockExtention);
151 if (lckFiles != null) {
152 for (int i=0; i < lckFiles.length; i++) {
153 String name = null;
154 try {
155 name = lckFiles[i].getCanonicalPath();
156 }
157 catch (IOException ex) {
158 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".getNewFiles", " could not get the canonical name of file '" + files[i].getName() + "'");
159 }
160 int pos = -1;
161 if (this.lockExt != null)
162 pos = (isFileNameCasesensitive() ? name.lastIndexOf(this.lockExt) : name.toUpperCase().lastIndexOf(this.lockExt.toUpperCase()));
163 if (pos < 0)
164 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_CONFIGURATION, ME, "can not handle lckExtention '*" + this.lockExt + "'");
165 this.lockFiles.add(name.substring(0, pos));
166 }
167 }
168 }
169
170 Map map = new HashMap(files.length);
171 for (int i=0; i < files.length; i++) {
172 try {
173 String name = files[i].getCanonicalPath();
174 if (files[i].isFile()) {
175 boolean endsWithLockExt = false;
176 if (this.lockExt != null)
177 endsWithLockExt = (isFileNameCasesensitive() ? name.endsWith(this.lockExt) : name.toUpperCase().endsWith(this.lockExt.toUpperCase()));
178 if (this.lockExtention == null || (!this.lockFiles.contains(name) && !endsWithLockExt))
179 map.put(name, files[i]);
180 }
181 }
182 catch (IOException ex) {
183 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".getNewFiles", " could not get the canonical name of file '" + files[i].getName() + "'");
184 }
185 }
186 return map;
187 }
188
189 /**
190 * On Windows sometimes the file is not deleted (even if the stream.close() were called before)
191 * We try as long until the file is away
192 * See http://forum.java.sun.com/thread.jspa?forumID=4&threadID=158689
193 * @param tempFile
194 * @return true if successfully deleted
195 */
196 private boolean deleteFile(File tempFile) {
197 if (!tempFile.exists())
198 return true;
199 final int MAX = 100;
200 boolean warn = false;
201 int i=0;
202 for (i=0; i<MAX; i++) {
203 if (!tempFile.delete()) {
204 warn = true;
205 if (!tempFile.exists()) // calling double delete fails, so check here
206 break;
207 if (i == 0)
208 log.fine(ME+": Deleting file " + tempFile.getAbsolutePath() + " failed");
209 System.gc();
210 if (!tempFile.delete()) {
211 if (i == 0)
212 log.warning(ME+": Deleting file " + tempFile.getAbsolutePath() + " failed even after GC");
213 try {
214 Thread.sleep(100);
215 } catch (InterruptedException e) {
216 }
217 }
218 else
219 break;
220 }
221 else
222 break;
223 }
224 if (i >= MAX) {
225 log.severe(ME+": Deleting file " + tempFile.getAbsolutePath() + " failed, giving up");
226 return false;
227 }
228 else {
229 if (warn)
230 log.info(ME+": Deleting file " + tempFile.getAbsolutePath() + " finally succeeded after " + (i+1) + " tries");
231 return true;
232 }
233 }
234
235 public static boolean isFileNameCasesensitive() {
236 String osName = System.getProperty("os.name");
237 if (osName == null)
238 return true;
239 if (osName.startsWith("Windows"))
240 return false;
241 return true;
242 }
243
244 /**
245 * Returns false if the info object is null, if the size is zero or
246 * if it has not passed sufficient time since the last change.
247 *
248 * @param info
249 * @param currentTime
250 * @return
251 */
252 private boolean isReady(FileInfo info, long currentTime) {
253 if (info == null)
254 return false;
255 //if (info.getSize() < 1L)
256 // return false;
257 if (this.lockExtention != null) {
258 return !this.lockFiles.contains(info.getName());
259 }
260 long delta = currentTime - info.getLastChange();
261 if (log.isLoggable(Level.FINEST)) {
262 log.finest(ME+": isReady '" + info.getName() + "' delta='" + delta + "' constant='" + this.delaySinceLastFileChange + "'");
263 }
264 return delta > this.delaySinceLastFileChange;
265 }
266
267 private TreeSet prepareEntries(File directory, Map existingFiles) {
268 if (log.isLoggable(Level.FINER))
269 log.finer(ME+": prepareEntries");
270
271 TreeSet chronologicalSet = new TreeSet(new FileComparator());
272 if (existingFiles == null || existingFiles.size() < 1) {
273 if (log.isLoggable(Level.FINEST)) {
274 log.finest(ME+": prepareEntries: nothing to do");
275 }
276 }
277 Iterator iter = existingFiles.values().iterator();
278 long currentTime = System.currentTimeMillis();
279 while (iter.hasNext()) {
280 FileInfo info = (FileInfo)iter.next();
281
282 if (isReady(info, currentTime)) {
283 chronologicalSet.add(info);
284 }
285 }
286 return chronologicalSet;
287 }
288
289 /**
290 * It updates the existing list of files:
291 *
292 * - if a file which previously existed is not found in the new list anymore it is deleted
293 * - new files are added to the list
294 * - if something has changed (timestamp or size, then the corresponding info object is touched
295 *
296 * @param existingFiles
297 * @param newFiles
298 */
299 private void updateExistingFiles(Map existingFiles, Map newFiles) {
300 Iterator iter = existingFiles.entrySet().iterator();
301 Set toRemove = new HashSet();
302 // scan all exising files: if some not found in new delete, otherwise
303 // update. At the end newFiles will only contain really new files
304 while (iter.hasNext()) {
305 Map.Entry existingEntry = (Map.Entry)iter.next();
306 Object key = existingEntry.getKey();
307 File newFile = (File)newFiles.get(key);
308 if (newFile == null) { // the file has been deleted: remove it from the list
309 if (toRemove == null)
310 toRemove = new HashSet();
311 toRemove.add(key);
312 }
313 else { // if still exists, then update
314 FileInfo existingInfo = (FileInfo)existingEntry.getValue();
315 existingInfo.update(newFile, log);
316 newFiles.remove(key);
317 }
318 }
319 // remove
320 if (toRemove != null && toRemove.size() > 0) {
321 String[] keys = (String[])toRemove.toArray(new String[toRemove.size()]);
322 for (int i=0; i < keys.length; i++) {
323 log.warning(ME+": the file '" + keys[i] + "' has apparently been removed from the outside: will not send it. No further action required");
324 existingFiles.remove(keys[i]);
325 }
326 }
327 // now we only have new files to process
328 iter = newFiles.values().iterator();
329 while (iter.hasNext()) {
330 File file = (File)iter.next();
331 FileInfo info = new FileInfo(file, log);
332 existingFiles.put(info.getName(), info);
333 }
334 }
335
336 /**
337 * Gets all entries which are ready to be sent (i.e. to be published)
338 *
339 * @return all entries as a TreeSet. Elements in the set are of type
340 * FileInfo
341 *
342 * @throws XmlBlasterException if the application has no read or write
343 * rights on the directory
344 */
345 Set getEntries() throws XmlBlasterException {
346 if (log.isLoggable(Level.FINER))
347 log.finer(ME+": getEntries");
348 Map newFiles = getNewFiles(this.directory);
349 updateExistingFiles(this.directoryEntries, newFiles);
350 return prepareEntries(this.directory, this.directoryEntries);
351 }
352
353 /**
354 * Removes the specified entry from the map. This method does also remove
355 * the entry from the file system or it moves it to the requested directory.
356 * If for some reason this is not
357 * possible, then an exception is thrown.
358 *
359 * @param entryName the name of the entry to remove.
360 * @return false if the entry was not found
361 * @throws XmlBlasterException
362 */
363 void deleteOrMoveEntry(final String entryName, boolean success) throws XmlBlasterException {
364 try {
365 if (log.isLoggable(Level.FINER))
366 log.finer(ME+": removeEntry '" + entryName + "'");
367 File file = new File(entryName);
368 if (!file.exists()) {
369 log.warning(ME+": removeEntry: '" + entryName + "' does not exist on the file system: I will only remove it from my list");
370 this.directoryEntries.remove(entryName);
371 return;
372 }
373
374 if (file.isDirectory())
375 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".removeEntry", "'" + entryName + "' is a directory");
376 if (!file.canWrite())
377 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".removeEntry", "no rights to write to '" + entryName + "'");
378
379 if (success && this.sentDirectory == null || !success && this.discardedDirectory == null) {
380 if (deleteFile(file)) {
381 this.directoryEntries.remove(entryName);
382 return;
383 }
384 else {
385 throw new XmlBlasterException(this.global, ErrorCode.INTERNAL_UNKNOWN, ME + ".removeEntry", "could not remove entry '" + file.getName() + "': retrying");
386 }
387 }
388 if (success) { // then do a move
389 moveTo(file, entryName, this.sentDirectory);
390 this.directoryEntries.remove(entryName);
391 }
392 else {
393 moveTo(file, entryName, this.discardedDirectory);
394 this.directoryEntries.remove(entryName);
395 }
396 }
397 catch (XmlBlasterException ex) {
398 throw ex;
399 }
400 catch (Throwable ex) {
401 throw new XmlBlasterException(this.global, ErrorCode.INTERNAL_UNKNOWN, ME + ".removeEntry", "", ex);
402 }
403 }
404
405 private void moveTo(File file, String origName, File destinationDirectory) throws XmlBlasterException {
406 if (!destinationDirectory.exists())
407 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".removeEntry", "'" + destinationDirectory.getName() + "' does not exist");
408 if (!destinationDirectory.isDirectory())
409 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".removeEntry", "'" + destinationDirectory.getName() + "' is not a directory");
410 if (!destinationDirectory.canRead())
411 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".removeEntry", "no rights to read to '" + destinationDirectory.getName() + "'");
412 if (!destinationDirectory.canWrite())
413 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".removeEntry", "no rights to write to '" + destinationDirectory.getName() + "'");
414
415 if (log.isLoggable(Level.FINE)) log.fine(ME+": File " + file.getAbsolutePath() + " moving to " + destinationDirectory.getAbsolutePath() + ", copyOnMove=" + copyOnMove);
416 String relativeName = FileInfo.getRelativeName(file.getName());
417 try {
418 File destinationFile = new File(destinationDirectory, relativeName);
419 if (destinationFile.exists()) {
420 boolean ret = deleteFile(destinationFile);
421 if (!ret)
422 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".moveTo", "could not delete the existing file '" + destinationFile.getCanonicalPath() + "' to '" + destinationDirectory.getName() + "' before moving avay '" + relativeName + "' after processing");
423 }
424 if (copyOnMove) {
425 InputStream inputStream = file.toURL().openStream();
426 BufferedInputStream bis = new BufferedInputStream(inputStream);
427 try {
428 FileOutputStream os = new FileOutputStream(destinationFile);
429 try {
430 long length = file.length();
431 long remaining = length;
432 final int BYTE_LENGTH = 100000; // For the moment it is hardcoded
433 byte[] buf = new byte[BYTE_LENGTH];
434 while (remaining > 0) {
435 int tot = bis.read(buf);
436 remaining -= tot;
437 os.write(buf, 0, tot);
438 }
439 }
440 finally {
441 try { os.close(); } catch (Throwable e) {}
442 }
443 }
444 finally {
445 try { bis.close(); } catch (Throwable e) {}
446 try { inputStream.close(); } catch (Throwable e) {}
447 }
448 String name = file.getAbsolutePath();
449 boolean deleted = deleteFile(file);
450 if (deleted) {
451 if (log.isLoggable(Level.FINE)) log.fine(ME+": File " + name + " is successfully deleted, copyOnMove=" + copyOnMove);
452 }
453 else {
454 log.warning(ME+": File " + name + " delete call failed: deleted=" + deleted + ", copyOnMove=" + copyOnMove + " exists=" + file.exists());
455 }
456 }
457 else {
458 boolean ret = file.renameTo(destinationFile);
459 if (!ret) {
460 File orig = new File(origName);
461 if (orig.exists()) {
462 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".moveTo", "Could not move the file '" + relativeName + "' to '" + destinationDirectory.getName() + "' reason: could it be that the destination is not a local file system ? try the flag 'copyOnMove='true' (see http://www.xmlblaster.org/xmlBlaster/doc/requirements/client.filepoller.html");
463 }
464 else {
465 File dest = new File(destinationDirectory, relativeName);
466 if (!dest.exists()) {
467 log.warning(ME+": Removed published file '" + origName + "' but couldn't create backup '" + destinationDirectory.getName() + "' (see http://www.xmlblaster.org/xmlBlaster/doc/requirements/client.filepoller.html");
468 }
469 else {
470 log.warning(ME+": Published file '" + origName + "' is already moved to backup '" + destinationDirectory.getName() + "' but java tells us it couldn't be moved, this is strange.");
471 }
472 }
473 }
474 }
475 }
476 catch (XmlBlasterException e) {
477 throw e;
478 }
479 catch (Throwable ex) {
480 log.warning(ME + ": Could not move the file '" + relativeName + "' to '" + destinationDirectory.getName() + "' reason: " + ex.toString());
481 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".moveTo", "could not move the file '" + relativeName + "' to '" + destinationDirectory.getName() + "' reason: ", ex);
482 }
483 }
484
485
486 /**
487 * Gets the content from the specified file as a byte[]. If this is
488 * not possible it will throw an exception.
489 *
490 * @param info
491 * @return
492 * @throws XmlBlasterException
493 */
494 public byte[] getContent(FileInfo info) throws XmlBlasterException {
495 String entryName = info.getName();
496 if (log.isLoggable(Level.FINER))
497 log.finer(ME+": getContent '" + entryName + "'");
498 File file = new File(entryName);
499 if (!file.exists()) {
500 log.warning(ME+": getContent: '" + entryName + "' does not exist on the file system: not sending anything");
501 this.directoryEntries.remove(entryName);
502 return null;
503 }
504 if (file.isDirectory())
505 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".getContent", "'" + entryName + "' is a directory");
506 if (!file.canWrite())
507 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".getContent", "no rights to write from '" + entryName + "'");
508
509 try {
510 int toRead = (int)info.getSize();
511 int offset = 0;
512 int tot = 0;
513
514 byte[] ret = new byte[toRead];
515 FileInputStream fis = new FileInputStream(entryName);
516 BufferedInputStream bis = new BufferedInputStream(fis);
517
518 while (tot < toRead) {
519 int available = bis.available();
520 if (available > 0) {
521 int read = bis.read(ret, offset, available);
522 tot += read;
523 }
524 else {
525 try {
526 Thread.sleep(5L);
527 }
528 catch (Exception e) {}
529 }
530 }
531 return ret;
532 }
533 catch (IOException ex) {
534 throw new XmlBlasterException(this.global, ErrorCode.RESOURCE_FILEIO, ME + ".getContent", "", ex);
535 }
536 catch (Throwable ex) {
537 throw new XmlBlasterException(this.global, ErrorCode.INTERNAL_UNKNOWN, ME + ".removeEntry", "", ex);
538 }
539 }
540
541 /** java org.xmlBlaster.client.filepoller.DirectoryManager -path /tmp/filepoller -filter "*.xml" -filterType simple */
542 public static void main(String[] args) {
543 try {
544 Global global = new Global(args);
545 String path = global.get("path", ".", null, null);
546 File directory = new File(path);
547 String filter = global.get("filter", "*.txt", null, null);
548 String filterType = global.get("filterType", "simple", null, null);
549 boolean trueRegex = false;
550 if ("regex".equalsIgnoreCase(filterType))
551 trueRegex = true;
552 System.out.println("-----------Configuration:-------------------------");
553 System.out.println("Directory to look into: '" + directory.getAbsolutePath() + "'");
554 System.out.println("The " + filterType + " filter is '" + filter + "'");
555 System.out.println("");
556 System.out.println("-----------Matching Results:----------------------");
557 FilenameFilter fileFilter = new FilenameFilter(filter, trueRegex);
558 File[] files = directory.listFiles(fileFilter);
559 if (files == null || files.length < 1) {
560 System.out.println("");
561 System.out.println("WARN: no files found matching the " + filterType + " expression '" + filter + "'");
562 System.out.println("");
563 System.exit(0);
564 }
565 for (int i=0; i < files.length; i++) {
566 System.out.println("file[" + i + "] = " + files[i].getName());
567 }
568 if (files.length > 0) {
569 System.out.println("");
570 System.out.println("no more files found");
571 }
572
573 }
574 catch (Exception ex) {
575 ex.printStackTrace();
576 }
577 }
578
579
580 }
syntax highlighted by Code2HTML, v. 0.9.1