From ba0b66ce54f33e18cf7acf0d7e4ef3f073745bbf Mon Sep 17 00:00:00 2001 From: moparisthebest Date: Thu, 9 Feb 2017 22:58:05 -0500 Subject: [PATCH] Initial commit --- .gitignore | 4 + license.txt | 165 ++++++++++++++++++ pom.xml | 58 ++++++ readme.md | 5 + .../filelist/ByteArrayConverter.java | 10 ++ .../filelist/LongConverter40Bit.java | 35 ++++ .../com/moparisthebest/filelist/Main.java | 48 +++++ .../filelist/RandomAccessFileList.java | 101 +++++++++++ .../filelist/SignedLongConverter.java | 87 +++++++++ .../filelist/UnsignedLongConverter.java | 34 ++++ 10 files changed, 547 insertions(+) create mode 100644 .gitignore create mode 100644 license.txt create mode 100755 pom.xml create mode 100644 readme.md create mode 100644 src/main/java/com/moparisthebest/filelist/ByteArrayConverter.java create mode 100644 src/main/java/com/moparisthebest/filelist/LongConverter40Bit.java create mode 100644 src/main/java/com/moparisthebest/filelist/Main.java create mode 100644 src/main/java/com/moparisthebest/filelist/RandomAccessFileList.java create mode 100644 src/main/java/com/moparisthebest/filelist/SignedLongConverter.java create mode 100644 src/main/java/com/moparisthebest/filelist/UnsignedLongConverter.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..40d0ced --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea/ +*.iml +target/ +out.xml diff --git a/license.txt b/license.txt new file mode 100644 index 0000000..65c5ca8 --- /dev/null +++ b/license.txt @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/pom.xml b/pom.xml new file mode 100755 index 0000000..b56a169 --- /dev/null +++ b/pom.xml @@ -0,0 +1,58 @@ + + 4.0.0 + + com.moparisthebest + filelists + 0.0.1 + jar + + filelists + + filelists + + https://github.com/moparisthebest/filelists + + + + + + + + Travis Burtrum + http://www.moparisthebest.com/ + + + + + + GNU LESSER GENERAL PUBLIC LICENSE, Version 3 + http://www.gnu.org/licenses/lgpl.html + + + + + compile + ${project.artifactId} + + + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + true + + -Xlint + + + + + + + + scm:git:git@github.com:moparisthebest/filelists.git + scm:git:git@github.com:moparisthebest/filelists.git + git@github.com:moparisthebest/filelists.git + + diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..2694989 --- /dev/null +++ b/readme.md @@ -0,0 +1,5 @@ +Java List implementation over files + +I needed a List 99,999,999,999 elements long, which wouldn't fit in ram of course, but could fit in 500gb as a 40-bit unsigned integer, these classes actually let you implement any List backed by a RandomAccessFile. + +Enjoy! \ No newline at end of file diff --git a/src/main/java/com/moparisthebest/filelist/ByteArrayConverter.java b/src/main/java/com/moparisthebest/filelist/ByteArrayConverter.java new file mode 100644 index 0000000..8096536 --- /dev/null +++ b/src/main/java/com/moparisthebest/filelist/ByteArrayConverter.java @@ -0,0 +1,10 @@ +package com.moparisthebest.filelist; + +/** + * Created by mopar on 2/9/17. + */ +public interface ByteArrayConverter { + int numBytes(); + T fromBytes(final byte[] buff); + void toBytes(final T o, final byte[] buff); +} diff --git a/src/main/java/com/moparisthebest/filelist/LongConverter40Bit.java b/src/main/java/com/moparisthebest/filelist/LongConverter40Bit.java new file mode 100644 index 0000000..2c4410a --- /dev/null +++ b/src/main/java/com/moparisthebest/filelist/LongConverter40Bit.java @@ -0,0 +1,35 @@ +package com.moparisthebest.filelist; + +/** + * Created by mopar on 2/9/17. + */ +public class LongConverter40Bit implements ByteArrayConverter { + + public static final ByteArrayConverter instance = new LongConverter40Bit(); + + private LongConverter40Bit() { + } + + @Override + public int numBytes() { + return 5; + } + + @Override + public Long fromBytes(final byte[] buffer) { + return (((long) buffer[4] & 0xFFL) << 32) + | (((long) buffer[3] & 0xFFL) << 24) + | (((long) buffer[2] & 0xFFL) << 16) + | (((long) buffer[1] & 0xFFL) << 8) + | ((long) buffer[0] & 0xFFL); + } + + @Override + public void toBytes(final Long l, final byte[] buffer) { + buffer[0] = (byte) ((l) & 0xFF); + buffer[1] = (byte) ((l >> 8) & 0xFF); + buffer[2] = (byte) ((l >> 16) & 0xFF); + buffer[3] = (byte) ((l >> 24) & 0xFF); + buffer[4] = (byte) ((l >> 32) & 0xFF); + } +} diff --git a/src/main/java/com/moparisthebest/filelist/Main.java b/src/main/java/com/moparisthebest/filelist/Main.java new file mode 100644 index 0000000..ffcfc11 --- /dev/null +++ b/src/main/java/com/moparisthebest/filelist/Main.java @@ -0,0 +1,48 @@ +package com.moparisthebest.filelist; + +import java.io.IOException; +import java.util.Date; +import java.util.List; + +/** + * Created by mopar on 2/9/17. + */ +public class Main { + public static void main(String[] args) throws IOException { + System.out.println("str " + new Date()); + //final List list = new RandomAccessFileList<>("/home/mopar/raf2.list", new UnsignedLongConverter(5)); + final List list = new RandomAccessFileList<>("/home/mopar/raf2.list", LongConverter40Bit.instance); + /* + System.out.println(list.get(0)); + System.out.println(((RandomAccessFileList)list).longSize()); + System.out.println(((RandomAccessFileList)list).get(((RandomAccessFileList)list).longSize() - 1)); + */ + //list.clear(); + /* + long max = Integer.MAX_VALUE + 1000L; + for(long l = 0; l < max; ++l) + list.add(l); + System.out.println("yay " + new Date()); + for(long l : list) + System.out.println(l); + */ + //if(true) return; + System.out.println(Integer.MAX_VALUE); + System.out.println(list); + list.add(5L); + System.out.println(list); + System.out.println(list.get(0)); + list.add(99999999999L); + System.out.println(list.get(1)); + list.add(6L); + System.out.println(list); + for(long l : list) + System.out.println(l); + list.sort(Long::compareTo); + System.out.println(list); + for(long l = 0; l < 1000; ++l) + list.add(l); + list.sort(Long::compareTo); + System.out.println(list); + } +} diff --git a/src/main/java/com/moparisthebest/filelist/RandomAccessFileList.java b/src/main/java/com/moparisthebest/filelist/RandomAccessFileList.java new file mode 100644 index 0000000..89c4633 --- /dev/null +++ b/src/main/java/com/moparisthebest/filelist/RandomAccessFileList.java @@ -0,0 +1,101 @@ +package com.moparisthebest.filelist; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.AbstractList; +import java.util.Objects; + +/** + * Created by mopar on 2/9/17. + */ +public class RandomAccessFileList extends AbstractList { + + private final RandomAccessFile raf; + private final byte[] buffer; + private final ByteArrayConverter bac; + + public RandomAccessFileList(final RandomAccessFile raf, final ByteArrayConverter bac) { + Objects.requireNonNull(raf); + Objects.requireNonNull(bac); + if(bac.numBytes() < 1) + throw new IllegalArgumentException("bytesPerEntry must be > 0"); + this.raf = raf; + this.buffer = new byte[bac.numBytes()]; + this.bac = bac; + } + + public RandomAccessFileList(final String name, final ByteArrayConverter bac) throws FileNotFoundException { + this(new RandomAccessFile(name, "rw"), bac); + } + + public RandomAccessFileList(final File file, final ByteArrayConverter bac) throws FileNotFoundException { + this(new RandomAccessFile(file, "rw"), bac); + } + + public T get(final int index) { + try { + return this.get((long) index); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public T get(final long index) throws IOException { + raf.seek(index * buffer.length); + if(raf.read(buffer) != buffer.length) + throw new IOException("no full buffer to read, corrupted file?"); + return bac.fromBytes(buffer); + } + + public int size() { + try { + return (int) this.longSize(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public long longSize() throws IOException { + return raf.length() / buffer.length; + } + + @Override + public boolean add(final T o) { + try { + raf.seek(raf.length()); + bac.toBytes(o, buffer); + raf.write(buffer); + return true; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public T set(final int index, final T o) { + try { + return this.set((long)index, o); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public T set(final long index, final T o) throws IOException { + final T ret = get(index); + raf.seek(index * buffer.length); + bac.toBytes(o, buffer); + raf.write(buffer); + return ret; + } + + @Override + public void clear() { + try { + raf.setLength(0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/com/moparisthebest/filelist/SignedLongConverter.java b/src/main/java/com/moparisthebest/filelist/SignedLongConverter.java new file mode 100644 index 0000000..86d26ab --- /dev/null +++ b/src/main/java/com/moparisthebest/filelist/SignedLongConverter.java @@ -0,0 +1,87 @@ +package com.moparisthebest.filelist; + +/** + * Created by mopar on 2/9/17. + */ +public class SignedLongConverter implements ByteArrayConverter { + + private static final boolean debug = false; + + private final int numBytes; + + public SignedLongConverter(final int numBytes) { + this.numBytes = numBytes; + } + + @Override + public int numBytes() { + return this.numBytes; + } + + @Override + public Long fromBytes(final byte[] buffer) { + /* + if(true) + return (((long)buffer[0] << 56) + + ((long)(buffer[1] & 255) << 48) + + ((long)(buffer[2] & 255) << 40) + + ((long)(buffer[3] & 255) << 32) + + ((long)(buffer[4] & 255) << 24) + + ((buffer[5] & 255) << 16) + + ((buffer[6] & 255) << 8) + + ((buffer[7] & 255) << 0)); + */ + int y = 0, x = (buffer.length * 8) - 8; + if(debug) System.out.printf("l = (long)(buffer[%d] << %d);%n", y, x); + long l = (long)(buffer[y] << x); + x = x - 8; + if(x > 23) + for (++y; x > 23; x = x - 8, ++y) { + if(debug) System.out.printf("l += ((long)(buffer[%d] & 255) << %d);%n", y, x); + l += ((long)(buffer[y] & 255) << x); + } + for (; y < buffer.length; x = x - 8, ++y) { + if(debug) System.out.printf("l += (buffer[%d] & 255) << %d;%n", y, x); + l += (buffer[y] & 255) << x; + } + return l; + } + + @Override + public void toBytes(final Long l, final byte[] buffer) { + /* + writeBuffer[0] = (byte)(v >>> 56); + writeBuffer[1] = (byte)(v >>> 48); + writeBuffer[2] = (byte)(v >>> 40); + writeBuffer[3] = (byte)(v >>> 32); + writeBuffer[4] = (byte)(v >>> 24); + writeBuffer[5] = (byte)(v >>> 16); + writeBuffer[6] = (byte)(v >>> 8); + writeBuffer[7] = (byte)(v >>> 0); + */ + for(int y = 0, x = (buffer.length * 8) - 8; x > -1; x = x - 8, ++y) { + if(debug) System.out.printf("buffer[%d] = (byte) (l >>> %d);%n", y, x); + buffer[y] = (byte) (l >>> x); + } + } + + public static void main(String[] args) { + final ByteArrayConverter bac = new SignedLongConverter(5); + final byte[] buf = new byte[3]; // 5 99999999999L + //System.out.println(4294967296L); System.out.println(Integer.MAX_VALUE * 2L); if(true)return; + //System.out.println(Long.MIN_VALUE + 500); + //bac.toBytes(99999999999L, buf); System.out.println(java.util.Arrays.toString(buf)); System.out.println(bac.fromBytes(buf)); if(true) return; + for(long l = 0, c = 0; ; ++l) { + bac.toBytes(l, buf); + c = bac.fromBytes(buf); + if(l != c) { + System.out.printf("limit for %d bytes l = %d c = %d%n", buf.length, l, c); + return; + } + if(l == 99999999999L) { + System.out.println("yay"); + return; + } + } + } +} diff --git a/src/main/java/com/moparisthebest/filelist/UnsignedLongConverter.java b/src/main/java/com/moparisthebest/filelist/UnsignedLongConverter.java new file mode 100644 index 0000000..e55849e --- /dev/null +++ b/src/main/java/com/moparisthebest/filelist/UnsignedLongConverter.java @@ -0,0 +1,34 @@ +package com.moparisthebest.filelist; + +/** + * Created by mopar on 2/9/17. + */ +public class UnsignedLongConverter implements ByteArrayConverter { + + private final int numBytes; + + public UnsignedLongConverter(final int numBytes) { + this.numBytes = numBytes; + } + + @Override + public int numBytes() { + return this.numBytes; + } + + @Override + public Long fromBytes(final byte[] buffer) { + long l = 0; + for(int x = buffer.length - 1, y = (buffer.length * 8) - 8; x >= 0; --x, y = y - 8) { + l |= (((long) buffer[x] & 0xFFL) << y); + } + return l; + } + + @Override + public void toBytes(final Long l, final byte[] buffer) { + for(int x = 0, y = 0; x < buffer.length; ++x, y = y + 8) { + buffer[x] = (byte) ((l >> y) & 0xFF); + } + } +}