mime.database source code

1 /**
2  * Access to Shared MIME-info database.
3  * Authors:
4  *  $(LINK2 https://github.com/FreeSlave, Roman Chistokhodov)
5  * License:
6  *  $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
7  * Copyright:
8  *  Roman Chistokhodov, 2016
9  * See_Also: $(LINK2 https://www.freedesktop.org/wiki/Specifications/shared-mime-info-spec/, Shared MIME database specification)
10  */
11 
12 module mime.database;
13 
14 import std.range;
15 import std.typecons;
16 
17 import mime.store;
18 import mime.detector;
19 import mime.text;
20 import mime.inode;
21 
22 import mime.common : dataSizeToRead;
23 public import mime.type;
24 
25 /**
26  * High-level class for accessing Shared MIME-info database.
27  */
28 final class MimeDatabase
29 {
30     /// Options for $(D mimeTypeForFile)
31     enum Match
32     {
33         globPatterns = 1,   /// Match file name against glob patterns.
34         magicRules   = 2,   /// Match file content against magic rules. With MatchOptions.globPatterns flag it's used only in conflicts.
35         namespaceURI = 4, /// Try to clarify mime type in case it's XML.
36         inodeType = 8, /// Provide inode/* type for files other than regular files.
37         textFallback = 16, /// Provide $(D text/plain) fallback if data seems to be textual.
38         octetStreamFallback = 32, /// Provide $(D application/octet-stream) fallback if data seems to be binary.
39         emptyFileFallback = 64, ///Provide $(B application/x-zerosize) fallback if mime type can't be detected, but data is known to be zero size.
40         all = globPatterns|magicRules|namespaceURI|inodeType|textFallback|octetStreamFallback|emptyFileFallback ///Use all recipes to detect MIME type.
41     }
42 
43     /**
44      * Constructor based on MIME paths.
45      * It uses $(D mime.detectors.cache.MimeDetectorFromCache) as MIME type detector and $(D mime.stores.files.FilesMimeStore) as MIME type store.
46      * Params:
47      *  mimePaths = Range of paths to base mime directories where needed files will be read.
48      * See_Also: $(D mime.paths.mimePaths), $(D mime.detectors.cache.MimeDetectorFromCache), $(D mime.stores.files.FilesMimeStore)
49      */
50     this(Range)(Range mimePaths) if (isInputRange!Range && is(ElementType!Range : string))
51     {
52         import mime.stores.subtypexml;
53         import mime.detectors.cache;
54 
55         _store = new MediaSubtypeXmlStore(mimePaths);
56         _detector = new MimeDetectorFromCache(mimePaths);
57     }
58 
59     /**
60      * Create MimeDatabase object with given store and detector.
61      */
62     @safe this(IMimeStore store, IMimeDetector detector)
63     {
64         _store = store;
65         _detector = detector;
66     }
67 
68     /**
69      * Get MIME types store used in MimeDatabase instance.
70      */
71     IMimeStore store()
72     {
73         return _store;
74     }
75 
76     /**
77      * Get MIME type detector used in MimeDatabase instance.
78      */
79     IMimeDetector detector()
80     {
81         return _detector;
82     }
83 
84     /**
85      * Get MIME type for given fileName.
86      * See_Also: $(D mimeTypeForFile)
87      */
88     Rebindable!(const(MimeType)) mimeTypeForFileName(string fileName)
89     {
90         return findExistingAlternative(_detector.mimeTypesForFileName(fileName));
91     }
92 
93     /**
94      * Get MIME type for given data.
95      * Note: This does NOT provide any fallbacks like text/plain, application/octet-stream or application/x-zerosize.
96      * See_Also: $(D mimeTypeForFile)
97      */
98     Rebindable!(const(MimeType)) mimeTypeForData(const(void)[] data)
99     {
100         return findExistingAlternative(_detector.mimeTypesForData(data));
101     }
102 
103     /**
104      * Get MIME type for file and its data using methods describing in options.
105      * Params:
106      *  fileName = Name of file
107      *  data = Data chunk read from the file. It's not necessary to read the whole file.
108      *  options = Lookup options
109      */
110     Rebindable!(const(MimeType)) mimeTypeForFile(string fileName, const(void)[] data, Match options = Match.all)
111     {
112         return mimeTypeForFileImpl(fileName, data, options, true);
113     }
114 
115     /**
116      * Get MIME type for file using methods describing in options.
117      * File contents will be read automatically if needed.
118      */
119     Rebindable!(const(MimeType)) mimeTypeForFile(string fileName, Match options = Match.all)
120     {
121         return mimeTypeForFileImpl(fileName, null, options, false);
122     }
123 
124     private auto checkIfXml(string fileName, const(void)[] data, const bool dataPassed)
125     {
126         static import std.file;
127         import mime.xml : getXMLnamespaceFromData;
128         if (!dataPassed)
129         {
130             try {
131                 data = std.file.read(fileName, dataSizeToRead);
132             } catch(Exception e) {
133                 return rebindable(const(MimeType).init);
134             }
135         }
136         string namespaceURI = getXMLnamespaceFromData(cast(const(char)[])data);
137         if (namespaceURI.length)
138         {
139             auto name = _detector.mimeTypeForNamespaceURI(namespaceURI);
140             return mimeType(name, No.resolveAlias);
141         }
142         return rebindable(const(MimeType).init);
143     }
144 
145     private auto mimeTypeForFileImpl(string fileName, const(void)[] data, Match options, bool dataPassed)
146     {
147         auto type = mimeTypeForFileImplRef(fileName, data, options, dataPassed);
148         if ((options & Match.namespaceURI) != 0 && type && type.name == "application/xml")
149         {
150             auto xmlType = checkIfXml(fileName, data, dataPassed);
151             if (xmlType)
152                 return xmlType;
153         }
154         return type;
155     }
156 
157     private auto mimeTypeForFileImplRef(string fileName, ref const(void)[] data, Match options, ref bool dataPassed)
158     {
159         static import std.file;
160         import std.file : getSize;
161         import std.exception : collectException;
162 
163         if (data is null && (options & Match.inodeType)) {
164             string inodeType = inodeMimeType(fileName);
165             if (inodeType.length) {
166                 return mimeType(inodeType);
167             }
168         }
169 
170         const(char[])[] mimeTypes;
171         if (options & Match.globPatterns) {
172             mimeTypes = _detector.mimeTypesForFileName(fileName);
173             if (mimeTypes.length == 1) {
174                 auto type = mimeType(mimeTypes[0]);
175                 if (type !is null) {
176                     return type;
177                 }
178             }
179         }
180 
181         if ((options & Match.emptyFileFallback) && mimeTypes.length == 0) {
182             if (!dataPassed) {
183                 ulong size;
184                 auto e = collectException(fileName.getSize, size);
185                 if (e is null && size == 0) {
186                     return mimeType("application/x-zerosize");
187                 }
188             } else {
189                 if (data.length == 0) {
190                     return mimeType("application/x-zerosize");
191                 }
192             }
193         }
194 
195         if (!dataPassed && (options & (Match.magicRules|Match.textFallback|Match.octetStreamFallback))) {
196             try {
197                 data = std.file.read(fileName, dataSizeToRead);
198             } catch(Exception e) {
199                 //pass
200             }
201         }
202 
203         if (data.length && (options & Match.magicRules)) {
204             auto conflicts = _detector.mimeTypesForData(data);
205             auto type = findExistingAlternative(conflicts);
206             if (type !is null) {
207                 return type;
208             }
209         }
210         if (mimeTypes.length) {
211             auto type = findExistingAlternative(mimeTypes);
212             if (type) {
213                 return type;
214             }
215         }
216         if (data.length && (options & Match.textFallback) && isTextualData(data)) {
217             return mimeType("text/plain");
218         }
219         if (data.length && (options & Match.octetStreamFallback)) {
220             return mimeType("application/octet-stream");
221         }
222 
223         return rebindable(const(MimeType).init);
224     }
225 
226     private auto findExistingAlternative(const(char[])[] conflicts)
227     {
228         foreach(name; conflicts) {
229             auto type = mimeType(name);
230             if (type !is null) {
231                 return type;
232             }
233         }
234         return rebindable(const(MimeType).init);
235     }
236 
237     /**
238      * Get mime type by name or alias.
239      * Params:
240      *  nameOrAlias = MIME type name or alias.
241      *  resolve = Try to resolve alias if could not find MIME type with given name.
242      * Returns: $(D mime.type.MimeType) for given nameOrAlias, resolving alias if needed. Null if no mime type found.
243      */
244     Rebindable!(const(MimeType)) mimeType(const(char)[] nameOrAlias, Flag!"resolveAlias" resolve = Yes.resolveAlias)
245     {
246         if (nameOrAlias.length == 0) {
247             return rebindable(const(MimeType).init);
248         }
249         auto type = _store.mimeType(nameOrAlias);
250         if (type is null && resolve) {
251             auto resolved = _detector.resolveAlias(nameOrAlias);
252             if (resolved.length) {
253                 type = _store.mimeType(resolved);
254             }
255         }
256         return type;
257     }
258 
259 private:
260     IMimeStore _store;
261     IMimeDetector _detector;
262 }
263 
264 ///
265 unittest
266 {
267     import mime.stores.files;
268     import mime.detectors.cache;
269 
270     auto mimePaths = ["./test/mime", "./test/discard", "./test/nonexistent"];
271 
272     alias FilesMimeStore.Options FOptions;
273     FOptions foptions;
274     ubyte opt = FOptions.required;
275 
276     foptions.types = opt;
277     foptions.aliases = opt;
278     foptions.subclasses = opt;
279     foptions.icons = opt;
280     foptions.genericIcons = opt;
281     foptions.XMLnamespaces = opt;
282     foptions.globs2 = opt;
283     foptions.globs = opt;
284     foptions.magic = opt;
285     foptions.treemagic = opt;
286 
287     auto store = new FilesMimeStore(mimePaths, foptions);
288     assert(!store.byMimeType().empty);
289     auto detector = new MimeDetectorFromCache(mimePaths);
290 
291     assert(detector.mimeCaches().length == 2);
292     assert(detector.mimeTypeForFileName("sprite.spr").length);
293     assert(detector.mimeTypeForFileName("model01.mdl").length);
294     assert(detector.mimeTypeForFileName("liblist.gam").length);
295     assert(detector.mimeTypeForFileName("no.exist").empty);
296     assert(detector.mimeTypeForData("IDSP\x02\x00\x00\x00") == "image/x-hlsprite");
297     assert(detector.resolveAlias("application/nonexistent") is null);
298 
299     assert(detector.mimeTypeForNamespaceURI("http://www.w3.org/1999/ent") == "text/x-ent");
300     assert(detector.mimeTypeForNamespaceURI("nonexistent").empty);
301 
302     auto database = new MimeDatabase(store, detector);
303     assert(database.detector() is detector);
304     assert(database.store() is store);
305 
306     assert(database.mimeType(string.init) is null);
307 
308     auto imageSprite = database.mimeType("image/x-hlsprite");
309     auto appSprite = database.mimeType("application/x-hlsprite");
310     assert(database.mimeType("application/x-hlsprite", No.resolveAlias) is null);
311     assert(imageSprite !is null && imageSprite is appSprite);
312 
313     assert(database.detector().isSubclassOf("text/x-fgd", "text/plain"));
314     assert(!database.detector().isSubclassOf("text/x-fgd", "application/octet-stream"));
315 
316     auto fgdType = database.mimeTypeForFileName("name.fgd");
317     assert(fgdType !is null);
318     assert(fgdType.name == "text/x-fgd");
319 
320     //testing Match options
321     auto iqm = database.mimeTypeForFile("model.iqm", MimeDatabase.Match.globPatterns);
322     assert(iqm !is null);
323     assert(iqm.name == "application/x-iqm");
324 
325     auto spriteType = database.mimeTypeForFile("sprite.spr", MimeDatabase.Match.globPatterns);
326     assert(spriteType !is null);
327 
328     auto sprite32 = database.mimeTypeForFile("sprite.spr", "IDSP\x20\x00\x00\x00", MimeDatabase.Match.magicRules);
329     assert(sprite32 !is null);
330     assert(sprite32.name == "image/x-sprite32");
331 
332     auto zeroType = database.mimeTypeForFile("nonexistent", (void[]).init, MimeDatabase.Match.emptyFileFallback);
333     assert(zeroType !is null);
334     assert(zeroType.name == "application/x-zerosize");
335 
336     zeroType = database.mimeTypeForFile("test/emptyfile", MimeDatabase.Match.emptyFileFallback);
337     assert(zeroType !is null);
338     assert(zeroType.name == "application/x-zerosize");
339 
340     auto textType = database.mimeTypeForFile("test/mime/types", MimeDatabase.Match.textFallback);
341     assert(textType !is null);
342     assert(textType.name == "text/plain");
343 
344     auto dirType = database.mimeTypeForFile("test", MimeDatabase.Match.inodeType);
345     assert(dirType !is null);
346     assert(dirType.name == "inode/directory");
347 
348     auto octetStreamType = database.mimeTypeForFile("test/mime/mime.cache", MimeDatabase.Match.octetStreamFallback);
349     assert(octetStreamType !is null);
350     assert(octetStreamType.name == "application/octet-stream");
351 
352     assert(database.mimeTypeForFile("file.unknown", MimeDatabase.Match.globPatterns) is null);
353 
354     //testing data
355     auto hlsprite = database.mimeTypeForData("IDSP\x02\x00\x00\x00");
356     assert(hlsprite !is null);
357     assert(hlsprite.name == "image/x-hlsprite");
358 
359     auto qsprite = database.mimeTypeForData("IDSP\x01\x00\x00\x00");
360     assert(qsprite !is null);
361     assert(qsprite.name == "image/x-qsprite");
362 
363     auto q2sprite = database.mimeTypeForData("IDS2");
364     assert(q2sprite !is null);
365     assert(q2sprite.name == "image/x-q2sprite");
366 
367     //testing case-insensitive suffix
368     auto vpk = database.mimeTypeForFileName("pakdir.vpk");
369     assert(vpk !is null);
370     assert(vpk.name == "application/vnd.valve.vpk");
371 
372     vpk = database.mimeTypeForFileName("pakdir.VPK");
373     assert(vpk !is null);
374     assert(vpk.name == "application/vnd.valve.vpk");
375 
376     //testing generic glob
377     auto modelseq = database.mimeTypeForFileName("model01.mdl");
378     assert(modelseq !is null);
379     assert(modelseq.name == "application/x-hlmdl-sequence");
380     modelseq = database.mimeTypeForFileName("model01.MDL");
381     assert(modelseq !is null && modelseq.name == "application/x-hlmdl-sequence");
382 
383     auto generalGlob = database.mimeTypeForFileName("general_test_long_glob");
384     assert(generalGlob !is null);
385     assert(generalGlob.name == "application/x-general-long-glob");
386     assert(database.detector.mimeTypeForFileName("general_test_long_glob"));
387 
388     assert(!database.mimeTypeForFileName("pak1.PAK"));
389     assert(database.mimeTypeForFileName("pak1.pak"));
390 
391     //testing case-sensitive suffix
392     assert(database.mimeTypeForFileName("my.shader"));
393     assert(!database.mimeTypeForFileName("my.SHADER"));
394 
395     //testing literal
396     assert(database.mimeTypeForFileName("liblist.gam"));
397     assert(database.mimeTypeForFileName("makefile"));
398 
399     //testing discard glob
400     assert(!database.mimeTypeForFileName("GNUmakefile"));
401     assert(!database.detector.mimeTypeForFileName("GNUmakefile"));
402 
403     assert(!database.mimeTypeForFileName("file.qvm3"));
404     assert(!database.detector.mimeTypeForFileName("file.qvm3"));
405 
406     assert(!database.mimeTypeForFileName("model01.sequence"));
407     assert(!database.detector.mimeTypeForFileName("model01.sequence"));
408 
409     //testing discard magic
410     assert(!database.mimeTypeForData("PAK"));
411     assert(!database.detector.mimeTypeForData("PAK"));
412     assert(!database.mimeTypeForFileName("file.qwad"));
413 
414     //conflicts
415     assert(database.mimeTypeForFileName("file.jmf"));
416     assert(database.mimeTypeForData("PACK"));
417 
418     //xml
419     assert(database.mimeTypeForFileName("file.xml").name == "application/xml");
420     assert(database.mimeTypeForData("<?xml").name == "application/xml");
421     assert(database.mimeTypeForFile("file.xml", `<start-element xmlns="http://www.w3.org/1999/ent">`).name == "text/x-ent");
422 }