Sejumlah besar data diperlukan beberapa waktu lalu. DBA menyediakan file CVS, tetapi setiap file CVS berukuran beberapa GB, sehingga dimuat secara langsung. Basis data sangat lambat dan akan menyebabkan memori yang tidak mencukupi. Untuk mewujudkan fungsi ini, program untuk dengan cepat membagi file ditulis.
impor org.apache.log4j.logmanager; import org.apache.log4j.logger; impor java.io.*; impor java.util.*; impor java.util.concurrent.*; Public Class FilesPlitutil {private final static logger log = logManager.getLogger (filePlitutil.class); Private Static Final Long OriginFileSize = 1024 * 1024 * 100; // 100m Private Static Final Int BlockfileSize = 1024 * 1024 * 64; // Untuk mencegah pemisah Cina yang kacau, Anda harus mengambil 2 ke N Power/ ** * CVS File Pemisah */ Private Static Final Char CvsSeparator = '^'; public static void main (string args []) {long start = system.currentTimeMillis (); coba {string filename = "d: //csvtest//aa.csv"; File sourceFile = file baru (nama file); if (sourceFile.length ()> = OriginFileSize) {string cvsfileName = filename.replaceall ("/////", "/"); FileSplitutil FileSplitutil = new FileSplitutil (); Daftar <String> Parts = FileSplitutil.splitBysize (cvsfileName, blockfileSize); untuk (bagian string: bagian) {System.out.println ("PartName adalah:"+bagian); }} System.out.println ("Panjang File Total" + SourceFile.Length () + ", waktu untuk membagi file:" + (System.CurrentTimeMillis () - mulai) + "ms."); } catch (Exception e) {log.info (e.getStackTrace ()); } } /** * Split file* * @param fileName The complete file name to be split* @param byteSize Split by how many bytes* @return List of split file names*/ public List<String> splitBySize(String fileName, int byteSize) throws IOException, InterruptedException { List<String> parts = new ArrayList<String>(); File file = file baru (nama file); int count = (int) math.ceil (file.length () / (ganda) bytesize); int countlen = (count + "") .length (); RandomAccessFile RAF = RandomAccessFile baru (nama file, "r"); long totallen = raf.length (); Latch Countdownlatch = Countdownlatch baru (Count); untuk (int i = 0; i <count; i ++) {string partFileName = file.getPath ()+"." + leftpad ((i + 1) + "", countlen, '0') + ".cvs"; int readsize = bytesize; Long startPos = (panjang) i * bytesize; long nextpos = (long) (i+1) * bytesize; if (nextPos> totallen) {readsize = (int) (totallen-startpos); } baru splitrunnable (baca, startpos, partfileName, file, latch) .run (); parts.add (partfilename); } latch.Await (); // Menunggu semua file ditulis // baris dapat dipotong selama pemotongan, dan semua file split dapat diproses, mergerow (bagian); bagian kembali; } / ** * Pemrosesan segmen runnable * * @author supeidong * / kelas privat splitRunnable mengimplementasikan runnable {int bytesize; String partfileName; File originfile; startpos yang lama; Latch Countdownlatch; public splitRunnable (int bytesize, startPos lama, string partfileName, file originfile, countdownlatch latch) {this.startpos = startpos; this.bytesize = bytesize; this.partFileName = partfileName; this.origInfile = Originfile; this.latch = latch; } public void run () {randomAccessFile rfile; Outputstream OS; coba {rfile = new randomAccessFile (Originfile, "r"); byte [] b = byte baru [bytesize]; rfile.seek (startpos); // pindahkan pointer ke awal setiap "segmen" int s = rfile.read (b); OS = FileOutputStream baru (PartFileName); os.write (b, 0, s); os.flush (); os.close (); latch.countdown (); } catch (ioException e) {log.error (e.getMessage ()); latch.countdown (); }}} / ** * Gabungan garis pemotongan * * @param bagian * / private void mergerow (daftar <string> bagian) {daftar <partFile> Partsfiles = ArrayList baru <partFile> (); coba {// rakit objek tabel split untuk (int i = 0; i <parts.size (); i ++) {string partfileName = parts.get (i); File splitFileTemp = file baru (PartFileName); if (splitFileTemp.exists ()) {partfile partfile = new partfile (); BufferedReader Reader = New BufferedReader (InputStreamReader baru (FileInputStream baru (SplitFileTemp), "GBK")); String firstrow = reader.readline (); String secondrow = reader.readline (); String endrow = readLastline (partfileName); partfile.setPartFileName (partfileName); partfile.setFirStrow (firstrow); partfile.setendrow (endrow); if (i> = 1) {string prepartFile = parts.get (i - 1); String preendrow = readLastline (prepartFile); partfile.setFirstisfull (getCharCount (firstrow+preendrow)> getCharcount (secondrow)); } partfiles.add (partfile); reader.close (); }} // Tulis baris yang perlu digabungkan untuk (int i = 0; i <partfiles.size () - 1; i ++) {partfile partfile = partfiles.get (i); Partfile partfileNext = partfiles.get (i + 1); StringBuilder SB = StringBuilder baru (); if (partfileNext.getFirStISFull ()) {sb.append ("/r/n"); SB.Append (PartFileNext.GetFirStrow ()); } else {sb.append (partfileNext.getFirStrow ()); } writeLastline (partfile.getPartFileName (), sb.tostring ()); }} catch (exception e) {log.error (e.getMessage ()); }} / ** * Dapatkan berapa kali karakter muncul * @param s * @return * / private int getCharCount (string s) {int count = 0; untuk (int i = 0; i <s.length (); i ++) {if (s.charat (i) == cvsseparator) {count ++; }} return count; } / ** * Gunakan bufferedInputStream untuk membaca jumlah baris file * * @param fileName * @return * / public int getFilerow (string fileName) melempar ioException {inputStream is = new bufferedInputStream (FileInputStream baru (nama file)); byte [] c = byte baru [1024]; Int Count = 0; int readchars = 0; while ((readchars = is.read (c))! = -1) {for (int i = 0; i <readchars; ++ i) {if (c [i] == '/n') ++ count; }} is.close (); jumlah pengembalian; } / ** * Baca baris terakhir dari data * @param fileName * @return * @throws IoException * / Private String ReadLastline (String FileName) melempar IoException {// Gunakan RandomAccessFile, temukan baris data terakhir dari balik acak ACCHACCESSFILE RAF = New RandomAccessFile (fileNename, "R"); Long len = raf.length (); String lastline = ""; if (len! = 0l) {long pos = len - 1; while (pos> 0) {pos--; raf.seek (pos); if (raf.readbyte () == '/n') {lastline = raf.readline (); lastline = string baru (lastline.getBytes ("8859_1"), "gbk"); merusak; }}}} raf.close (); kembali garis terakhir; } /** * Modify the last line of data* @param fileName * @param lastString * @return * @throws IOException */ private void writeLastLine(String fileName,String lastString){ try { // Open a random access file stream and read and write according to the read and write method RandomAccessFile randomFile = new RandomAccessFile(fileName, "rw"); // panjang file, jumlah byte panjang filelength = randomFile.length (); // Pindahkan pointer file tulis ke akhir file. randomfile.seek (FileLength); // GBK harus ditambahkan di sini, jika tidak, acak hable.write (laststring.getbytes ("gbk")); randomfile.close (); } catch (ioException e) {log.error (e.getMessage ()); }} / ** * Isi kiri * * @param str * @param panjang * @param ch * @return * / string statis public leftpad (string str, panjang int, char ch) {if (str.length ()> = length) {return str; } char [] chs = char baru [panjang]; Arrays.fill (chs, ch); char [] src = str.tochararray (); System.ArrayCopy (SRC, 0, CHS, Length - Src.Length, Src.Length); mengembalikan string baru (CHS); } / *** Gabungan baris file internal file internal* / class partfile {private string partfileName; Private String Firstrow; Private String Endrow; firstisfull boolean pribadi; string publik getPartFileName () {return partfileName; } public void setPartFileName (String PartFileName) {this.partFileName = partFileName; } public String getFirStrow () {return firstrow; } public void setFirStrow (String firstrow) {this.firstrow = firstrow; } public String getEndRow () {return endrow; } public void setendrow (String endrow) {this.endrow = endrow; } public boolean getFirstiSfull () {return firstisfull; } public void setFirstiSfull (boolean firstisfull) {this.firstisfull = firstisfull; }}}Di atas adalah semua tentang artikel ini, saya harap akan sangat membantu bagi semua orang untuk belajar pemrograman Java.