Zimbra Desktop 2 Storage Migration: Difference between revisions
Davidfraser (talk | contribs) (Start of unofficial guide to migrating mail store to zd2) |
No edit summary |
||
(7 intermediate revisions by 3 users not shown) | |||
Line 1: | Line 1: | ||
= Zimbra Desktop 2 Storage Migration = | {{BC|Community Sandbox}} | ||
__FORCETOC__ | |||
<div class="col-md-12 ibox-content"> | |||
=Zimbra Desktop 2 Storage Migration= | |||
{{KB|{{Unsupported}}|{{ZCS 7.0}}|{{ZCS 6.0}}|}} | |||
{{Archive}}{{WIP}}= Zimbra Desktop 2 Storage Migration = | |||
== Rationale and Plan == | |||
These are unofficial notes on how the store for items in Zimbra Desktop 2 differs from that in Zimbra Desktop 1, designed to aid in migration. | These are unofficial notes on how the store for items in Zimbra Desktop 2 differs from that in Zimbra Desktop 1, designed to aid in migration. | ||
The | The rationale behind this is: | ||
* Zimbra Desktop 2 requires a totally fresh setup; no migration path from ZD1 is provided | |||
* I am living in a country with slow and expensive bandwidth | |||
* Our mail server is in a country with fast and cheap bandwidth | |||
So my plan is: | So my plan is: | ||
* Install Zimbra Desktop 2 somewhere with fast and cheap bandwidth to our mail server | |||
* Set up the same accounts as on Zimbra Desktop 1 | |||
* Use rsync to efficiently transfer the Zimbra Desktop 2 setup without having to retransmit all messages | |||
* Regenerate indexes instead of transferring them | |||
== Differences in Storage scheme == | |||
* Zimbra Desktop 2 uses ''sqlite'' instead of ''derby'' for the local database | |||
* Zimbra Desktop 2 uses gzip for all mail items over (approximately) 140kb; Zimbra Desktop 1 doesn't | |||
Complications that mean the installations don't exactly match: | Complications that mean the installations don't exactly match: | ||
* The file store uses different names for the same mail items | |||
* The indexes produced (for text searching) will be different (and can be fairly large) | |||
* The gzip compression doesn't seem to exactly match that produced by the command-line gzip tool at any setting, so we need to use the Java gzip classes to do compression | |||
* Zimbra Desktop 1 seems to have stored some strange items on my machine | |||
== Scripts == | == Scripts == | ||
Requirements: Python, Java, Bash | |||
These scripts should be saved to a common directory. Run <pre>make</pre> to compile the Java code. Run <pre>./upgradezd remote_host</pre> to do the upgrade (where <tt>$remote_host</tt> is the ssh connectin string for the remote host and user)... This '''will''' require manually adjustment of the scripts... | |||
=== Java gzip implementation === | |||
Save this as <tt>gzip.java</tt>: | |||
<pre> | |||
import java.util.zip.GZIPOutputStream; | |||
import java.io.FileOutputStream; | |||
import java.io.FileInputStream; | |||
import java.io.IOException; | |||
public class gzip { | |||
public static void main(String args[]) { | |||
if(args.length<=0) | |||
{ | |||
System.out.println("Please enter the valid file name"); | |||
} | |||
else | |||
{ | |||
try | |||
{ | |||
String inFilename = args[0]; | |||
String outFilename = args[1]; | |||
GZIPOutputStream out = new GZIPOutputStream(new FileOutputStream(outFilename)); | |||
FileInputStream in = new FileInputStream(inFilename); | |||
// Transfer bytes from the input file to the gzip output stream | |||
byte[] buf = new byte[1024]; | |||
int len; | |||
while ((len = in.read(buf)) > 0) | |||
{ | |||
out.write(buf, 0, len); | |||
} | |||
in.close(); | |||
out.finish(); | |||
out.close(); | |||
} | |||
catch(IOException e) | |||
{ | |||
System.out.println("Exception has been thrown" + e); | |||
} | |||
} | |||
} | |||
} | |||
</pre> | |||
=== Java gunzip implementation === | |||
Save this as <tt>gunzip.java</tt>: | |||
<pre> | |||
import java.util.zip.GZIPInputStream; | |||
import java.io.FileOutputStream; | |||
import java.io.FileInputStream; | |||
import java.io.IOException; | |||
public class gunzip { | |||
public static void main(String args[]) { | |||
if(args.length<=0) | |||
{ | |||
System.out.println("Please enter the valid file name"); | |||
} | |||
else | |||
{ | |||
try | |||
{ | |||
String inFilename = args[0]; | |||
String outFilename = args[1]; | |||
GZIPInputStream gzipInputStream = | |||
new GZIPInputStream(new FileInputStream(inFilename)); | |||
FileOutputStream out = new FileOutputStream(outFilename); | |||
// Transfer bytes from the compressed file to the output file | |||
byte[] buf = new byte[1024]; | |||
int len; | |||
while ((len = gzipInputStream.read(buf)) > 0) | |||
{ | |||
out.write(buf, 0, len); | |||
} | |||
gzipInputStream.close(); | |||
out.close(); | |||
} | |||
catch(IOException e) | |||
{ | |||
System.out.println("Exception has been thrown" + e); | |||
} | |||
} | |||
} | |||
} | |||
</pre> | |||
=== Makefile === | |||
Save this as <tt>Makefile</tt>: | |||
<pre> | |||
%.class: %.java | |||
javac $< | |||
%.manifest: %.java | |||
echo "Main-Class: $(shell basename $<)" > $@ | |||
%.jar: %.manifest %.class | |||
jar -cfm $@ $+ | |||
all: gunzip.jar gzip.jar | |||
</pre> | |||
=== Python renaming code === | |||
Save this as <tt>zd2rename.py</tt>: | |||
<pre> | |||
#!/usr/bin/env python | |||
import sys, os, re, logging | |||
ZD2_STORE = sys.argv[1] | |||
logging.getLogger().setLevel(logging.INFO) | |||
FNAME_RE = re.compile("([0-9]+)-([0-9]+)[.]msg") | |||
local_maps = {} | |||
for line in sys.stdin: | |||
line = line.strip() | |||
if not line: | |||
continue | |||
dirname, fname = os.path.dirname(line), os.path.basename(line) | |||
fmatch = FNAME_RE.match(fname) | |||
if not fmatch: | |||
logging.warning("Unexpected filename: %s", filename) | |||
continue | |||
prefix, suffix = fmatch.group(1), fmatch.group(2) | |||
# print "mv %s/%s-*.msg %s/%s" % (dirname, prefix, dirname, fname) | |||
zd2_dirname = os.path.join(ZD2_STORE, dirname) | |||
if dirname not in local_maps: | |||
if not os.path.isdir(zd2_dirname): | |||
logging.warning("Directory %s not found locally", dirname) | |||
local_maps[dirname] = None | |||
continue | |||
local_map = {} | |||
for local_fname in os.listdir(zd2_dirname): | |||
local_fmatch = FNAME_RE.match(local_fname) | |||
if local_fmatch: | |||
local_map.setdefault(local_fmatch.group(1), []).append(local_fname) | |||
local_maps[dirname] = local_map | |||
else: | |||
local_map = local_maps[dirname] | |||
if local_map is None: | |||
continue | |||
local_matches = local_map.get(prefix, []) | |||
if len(local_matches) == 1: | |||
local_src, local_target = os.path.join(zd2_dirname, local_matches[0]), os.path.join(zd2_dirname, fname) | |||
if local_src != local_target: | |||
logging.info("Renaming %s/%s to %s/%s", dirname, local_matches[0], dirname, fname) | |||
os.rename(local_src, local_target) | |||
elif len(local_matches) > 1: | |||
# cat all the matches onto the file | |||
logging.warning("Unexpected multiple match for %s/%s: %s", dirname, " ".join(local_matches)) | |||
open(os.path.join(zd2_dirname, fname), "wb").writelines(open(os.path.join(zd2_dirname, local_fname), "rb").read() for local_fname in local_matches) | |||
else: | |||
logging.info("File not found locally: %s/%s", dirname, fname) | |||
</pre> | |||
=== Upgrade controller script === | |||
Save this as <tt>upgradezd</pre>, then run: | |||
<pre>chmod a+x upgradezd</pre> | |||
Manually edit as required (see especially the <tt>TODO</pre> about store numbers). Stop Zimbra desktop on both sides before running the upgrade. | |||
Run with: | |||
<pre>./upgradezd $remote_host</pre> | |||
where <tt>$remote_host</tt> is the ssh connection string for the remote machine and user. | |||
<pre> | |||
#!/bin/bash | |||
zd1_dir=$HOME/zimbra/zdesktop | |||
zd2_dir=$HOME/zdesktop | |||
src_dir="`dirname "$0"`" | |||
src_dir="`cd "$src_dir" ; pwd`" | |||
zd1_store=$zd1_dir/store | |||
zd2_store=$zd2_dir/store | |||
mkdir -p $zd2_dir | |||
# initial transfer of store | |||
rsync -a $zd1_store/ $zd2_store/ | |||
# gzip of all large blobs | |||
( | |||
cd $zd2_store | |||
gzh=$'\x1f\x8b' | |||
find . -type f -name "*.msg" -size "+140k" -print0 | xargs -0 -n 1 -I % bash -c "gzh='$gzh' ; src_dir='$src_dir' ; "'{ h="`head -c 2 %`" ; [[ "$h" != "$gzh" ]] && echo % && java -jar $src_dir/gzip.jar % %.gz && mv %.gz % && ls -l % ; }' | |||
) | |||
# adjust store ids to match | |||
( | |||
cd $zd2_store | |||
# TODO: manually adjust these instructions so that your store names match | |||
mv 0/4 0/3 | |||
) | |||
# now rename to match names on the other side... | |||
remote_host="$1" | |||
[[ "$remote_host" == "" ]] && { echo syntax "$0" remote_host >&2 ; exit 1 ; } | |||
remote_dir=zdesktop | |||
remote_store=$remote_dir/store | |||
ssh $remote_host "cd $remote_store ; find . -type f -name '*.msg'" | tee $src_dir/remote-listing | python zd2rename.py "$zd2_store" | |||
rsync -avzP $remote_host:$remote_store/ $zd2_store/ | |||
# after having completed install | |||
rsync -avzP --exclude index $remote_host:$remote_store/ $zd2_dir/ | |||
rm -fr $zd2_dir/index/* | |||
# now start zimbra desktop 2 and regenerate indexes | |||
</pre> | |||
{{Article Footer|Zimbra Desktop 2|2/25/2010}} | |||
[[Category:Migration]] | |||
[[Category:Zimbra Desktop]] |
Latest revision as of 13:52, 13 July 2015
Zimbra Desktop 2 Storage Migration
Zimbra Desktop 2 Storage Migration
Rationale and Plan
These are unofficial notes on how the store for items in Zimbra Desktop 2 differs from that in Zimbra Desktop 1, designed to aid in migration.
The rationale behind this is:
- Zimbra Desktop 2 requires a totally fresh setup; no migration path from ZD1 is provided
- I am living in a country with slow and expensive bandwidth
- Our mail server is in a country with fast and cheap bandwidth
So my plan is:
- Install Zimbra Desktop 2 somewhere with fast and cheap bandwidth to our mail server
- Set up the same accounts as on Zimbra Desktop 1
- Use rsync to efficiently transfer the Zimbra Desktop 2 setup without having to retransmit all messages
- Regenerate indexes instead of transferring them
Differences in Storage scheme
- Zimbra Desktop 2 uses sqlite instead of derby for the local database
- Zimbra Desktop 2 uses gzip for all mail items over (approximately) 140kb; Zimbra Desktop 1 doesn't
Complications that mean the installations don't exactly match:
- The file store uses different names for the same mail items
- The indexes produced (for text searching) will be different (and can be fairly large)
- The gzip compression doesn't seem to exactly match that produced by the command-line gzip tool at any setting, so we need to use the Java gzip classes to do compression
- Zimbra Desktop 1 seems to have stored some strange items on my machine
Scripts
Requirements: Python, Java, Bash
These scripts should be saved to a common directory. Runmaketo compile the Java code. Run
./upgradezd remote_hostto do the upgrade (where $remote_host is the ssh connectin string for the remote host and user)... This will require manually adjustment of the scripts...
Java gzip implementation
Save this as gzip.java:
import java.util.zip.GZIPOutputStream; import java.io.FileOutputStream; import java.io.FileInputStream; import java.io.IOException; public class gzip { public static void main(String args[]) { if(args.length<=0) { System.out.println("Please enter the valid file name"); } else { try { String inFilename = args[0]; String outFilename = args[1]; GZIPOutputStream out = new GZIPOutputStream(new FileOutputStream(outFilename)); FileInputStream in = new FileInputStream(inFilename); // Transfer bytes from the input file to the gzip output stream byte[] buf = new byte[1024]; int len; while ((len = in.read(buf)) > 0) { out.write(buf, 0, len); } in.close(); out.finish(); out.close(); } catch(IOException e) { System.out.println("Exception has been thrown" + e); } } } }
Java gunzip implementation
Save this as gunzip.java:
import java.util.zip.GZIPInputStream; import java.io.FileOutputStream; import java.io.FileInputStream; import java.io.IOException; public class gunzip { public static void main(String args[]) { if(args.length<=0) { System.out.println("Please enter the valid file name"); } else { try { String inFilename = args[0]; String outFilename = args[1]; GZIPInputStream gzipInputStream = new GZIPInputStream(new FileInputStream(inFilename)); FileOutputStream out = new FileOutputStream(outFilename); // Transfer bytes from the compressed file to the output file byte[] buf = new byte[1024]; int len; while ((len = gzipInputStream.read(buf)) > 0) { out.write(buf, 0, len); } gzipInputStream.close(); out.close(); } catch(IOException e) { System.out.println("Exception has been thrown" + e); } } } }
Makefile
Save this as Makefile:
%.class: %.java javac $< %.manifest: %.java echo "Main-Class: $(shell basename $<)" > $@ %.jar: %.manifest %.class jar -cfm $@ $+ all: gunzip.jar gzip.jar
Python renaming code
Save this as zd2rename.py:
#!/usr/bin/env python import sys, os, re, logging ZD2_STORE = sys.argv[1] logging.getLogger().setLevel(logging.INFO) FNAME_RE = re.compile("([0-9]+)-([0-9]+)[.]msg") local_maps = {} for line in sys.stdin: line = line.strip() if not line: continue dirname, fname = os.path.dirname(line), os.path.basename(line) fmatch = FNAME_RE.match(fname) if not fmatch: logging.warning("Unexpected filename: %s", filename) continue prefix, suffix = fmatch.group(1), fmatch.group(2) # print "mv %s/%s-*.msg %s/%s" % (dirname, prefix, dirname, fname) zd2_dirname = os.path.join(ZD2_STORE, dirname) if dirname not in local_maps: if not os.path.isdir(zd2_dirname): logging.warning("Directory %s not found locally", dirname) local_maps[dirname] = None continue local_map = {} for local_fname in os.listdir(zd2_dirname): local_fmatch = FNAME_RE.match(local_fname) if local_fmatch: local_map.setdefault(local_fmatch.group(1), []).append(local_fname) local_maps[dirname] = local_map else: local_map = local_maps[dirname] if local_map is None: continue local_matches = local_map.get(prefix, []) if len(local_matches) == 1: local_src, local_target = os.path.join(zd2_dirname, local_matches[0]), os.path.join(zd2_dirname, fname) if local_src != local_target: logging.info("Renaming %s/%s to %s/%s", dirname, local_matches[0], dirname, fname) os.rename(local_src, local_target) elif len(local_matches) > 1: # cat all the matches onto the file logging.warning("Unexpected multiple match for %s/%s: %s", dirname, " ".join(local_matches)) open(os.path.join(zd2_dirname, fname), "wb").writelines(open(os.path.join(zd2_dirname, local_fname), "rb").read() for local_fname in local_matches) else: logging.info("File not found locally: %s/%s", dirname, fname)
Upgrade controller script
Save this as upgradezd, then run:chmod a+x upgradezdManually edit as required (see especially the TODO about store numbers). Stop Zimbra desktop on both sides before running the upgrade.
Run with:
./upgradezd $remote_host
where $remote_host is the ssh connection string for the remote machine and user.
#!/bin/bash zd1_dir=$HOME/zimbra/zdesktop zd2_dir=$HOME/zdesktop src_dir="`dirname "$0"`" src_dir="`cd "$src_dir" ; pwd`" zd1_store=$zd1_dir/store zd2_store=$zd2_dir/store mkdir -p $zd2_dir # initial transfer of store rsync -a $zd1_store/ $zd2_store/ # gzip of all large blobs ( cd $zd2_store gzh=$'\x1f\x8b' find . -type f -name "*.msg" -size "+140k" -print0 | xargs -0 -n 1 -I % bash -c "gzh='$gzh' ; src_dir='$src_dir' ; "'{ h="`head -c 2 %`" ; [[ "$h" != "$gzh" ]] && echo % && java -jar $src_dir/gzip.jar % %.gz && mv %.gz % && ls -l % ; }' ) # adjust store ids to match ( cd $zd2_store # TODO: manually adjust these instructions so that your store names match mv 0/4 0/3 ) # now rename to match names on the other side... remote_host="$1" [[ "$remote_host" == "" ]] && { echo syntax "$0" remote_host >&2 ; exit 1 ; } remote_dir=zdesktop remote_store=$remote_dir/store ssh $remote_host "cd $remote_store ; find . -type f -name '*.msg'" | tee $src_dir/remote-listing | python zd2rename.py "$zd2_store" rsync -avzP $remote_host:$remote_store/ $zd2_store/ # after having completed install rsync -avzP --exclude index $remote_host:$remote_store/ $zd2_dir/ rm -fr $zd2_dir/index/* # now start zimbra desktop 2 and regenerate indexes