Commit 9af0f80b authored by Pablo Iturrieta Rebolledo's avatar Pablo Iturrieta Rebolledo
Browse files

updated brier score and multi score figure

parent 0af670d8
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="PublishConfigData" autoUpload="Always" serverName="pciturri@rs1:22" remoteFilesAllowedToDisappearOnAutoupload="false"> <component name="PublishConfigData" autoUpload="Always" remoteFilesAllowedToDisappearOnAutoupload="false">
<serverData>
<paths name="pciturri@rs1:22">
<serverdata>
<mappings>
<mapping deploy="/home/pciturri/pycharmprojects/italy-experiment-testing" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
</serverData>
<option name="myAutoUpload" value="ALWAYS" /> <option name="myAutoUpload" value="ALWAYS" />
</component> </component>
</project> </project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.8 (venv)" project-jdk-type="Python SDK" /> <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7 (venv)" project-jdk-type="Python SDK" />
</project> </project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="PySciProjectComponent"> <component name="PySciProjectComponent">
<option name="PY_SCI_VIEW" value="true" />
<option name="PY_SCI_VIEW_SUGGESTED" value="true" /> <option name="PY_SCI_VIEW_SUGGESTED" value="true" />
</component> </component>
</project> </project>
\ No newline at end of file
...@@ -166,7 +166,7 @@ def run(use_saved=False, catalog_name='bollettino'): ...@@ -166,7 +166,7 @@ def run(use_saved=False, catalog_name='bollettino'):
get_ref_IG(Results[10]) get_ref_IG(Results[10])
if __name__ == '__main__': if __name__ == '__main__':
use_saved = False use_saved = True
catalogs = ['bollettino', 'emrcmt', 'horus'] catalogs = ['bollettino', 'emrcmt', 'horus']
for cat in catalogs: for cat in catalogs:
run(use_saved, cat) run(use_saved, cat)
......
...@@ -148,7 +148,7 @@ def run(use_saved=False, catalog_name='bollettino'): ...@@ -148,7 +148,7 @@ def run(use_saved=False, catalog_name='bollettino'):
if __name__ == '__main__': if __name__ == '__main__':
use_saved = True use_saved = True
catalogs = ['bollettino', 'emrcmt', 'horus'][:1] catalogs = ['bollettino', 'emrcmt', 'horus']
for cat in catalogs: for cat in catalogs:
run(use_saved, cat) run(use_saved, cat)
......
...@@ -27,7 +27,7 @@ def process_forecasts_batch(Model_batch, catalog, save_obj=False, load_obj=False ...@@ -27,7 +27,7 @@ def process_forecasts_batch(Model_batch, catalog, save_obj=False, load_obj=False
if not load_obj: if not load_obj:
Results_T = [] Results_T = []
Results_W = [] Results_W = []
for f_i in Model_batch: for f_i in Model_batch:
T_ref_i = [] T_ref_i = []
W_ref_i = [] W_ref_i = []
for f_j in Model_batch: for f_j in Model_batch:
...@@ -35,9 +35,9 @@ def process_forecasts_batch(Model_batch, catalog, save_obj=False, load_obj=False ...@@ -35,9 +35,9 @@ def process_forecasts_batch(Model_batch, catalog, save_obj=False, load_obj=False
W_ref_i.append(poisson.w_test(f_j, f_i, catalog)) W_ref_i.append(poisson.w_test(f_j, f_i, catalog))
Results_T.append(T_ref_i) Results_T.append(T_ref_i)
Results_W.append(W_ref_i) Results_W.append(W_ref_i)
if save_obj: if save_obj:
with open(save_obj, 'wb') as obj: with open(save_obj, 'wb') as obj:
pickle.dump((Results_T, Results_W), obj) pickle.dump((Results_T, Results_W), obj)
else: else:
try: try:
with open(load_obj, 'rb') as obj: with open(load_obj, 'rb') as obj:
...@@ -81,7 +81,7 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False): ...@@ -81,7 +81,7 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False):
names = [i[0].sim_name[1] for i in Results[0]] names = [i[0].sim_name[1] for i in Results[0]]
T_value = np.array([[T_i.observed_statistic for T_i in Ref_model] for Ref_model in Results[0]]).T T_value = np.array([[T_i.observed_statistic for T_i in Ref_model] for Ref_model in Results[0]]).T
T_quantile = np.array([[T_i.test_distribution[0] for T_i in Ref_model] for Ref_model in Results[0]]).T T_quantile = np.array([[np.abs(T_i.quantile[0]) - T_i.quantile[1] for T_i in Ref_model] for Ref_model in Results[0]]).T
W_quantile = np.array([[W_i.quantile for W_i in Ref_model] for Ref_model in Results[1]]).T W_quantile = np.array([[W_i.quantile for W_i in Ref_model] for Ref_model in Results[1]]).T
score = np.sum(T_value, axis=1)/T_value.shape[0] score = np.sum(T_value, axis=1)/T_value.shape[0]
...@@ -102,11 +102,15 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False): ...@@ -102,11 +102,15 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False):
ax.set_yticklabels([names[i] for i in arg_ind], rotation='horizontal') ax.set_yticklabels([names[i] for i in arg_ind], rotation='horizontal')
ax.set_xticklabels([names[i] for i in arg_ind],rotation='vertical') ax.set_xticklabels([names[i] for i in arg_ind],rotation='vertical')
for n, i in enumerate(data_tq):
for m, j in enumerate(i):
if j > 0:
ax.scatter(n + 0.5, m + 0.5, marker='o', s=75, facecolor='None', edgecolor='black')
for n, i in enumerate(data_w): for n, i in enumerate(data_w):
for m, j in enumerate(i): for m, j in enumerate(i):
if j < p: if j < p:
ax.scatter(n + 0.5, m + 0.5,marker='o', s=2, color='black') ax.scatter(n + 0.5, m + 0.5, marker='o', s=5, color='black')
# plt.text(len(score) + 2.5, -0.2, '$G_t$', fontsize=20) # plt.text(len(score) + 2.5, -0.2, '$G_t$', fontsize=20)
# for i, j in enumerate([score[i] for i in arg_ind]): # for i, j in enumerate([score[i] for i in arg_ind]):
...@@ -114,8 +118,11 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False): ...@@ -114,8 +118,11 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False):
ax.set_title('Information Gain $T-$test') ax.set_title('Information Gain $T-$test')
legend_elements = [Line2D([0], [0], marker='o', lw=0, label='w-significant', legend_elements = [Line2D([0], [0], marker='o', lw=0, label='$W$-significant',
markerfacecolor='black', markeredgecolor='black', markersize=1)] markerfacecolor="black", markeredgecolor='black', markersize=4),
Line2D([0], [0], marker='o', lw=0, label='$T$-significant',
markerfacecolor="None", markeredgecolor='black', markersize=10)
]
fig.legend(handles=legend_elements, loc='lower right', fig.legend(handles=legend_elements, loc='lower right',
bbox_to_anchor=(0.79, 0.08, 0.2, 0.2), handletextpad=0) bbox_to_anchor=(0.79, 0.08, 0.2, 0.2), handletextpad=0)
plt.tight_layout() plt.tight_layout()
...@@ -123,9 +130,9 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False): ...@@ -123,9 +130,9 @@ def plot_all_results(Results, years, p=0.05, order=False, savepath=False):
plt.show() plt.show()
def run(use_saved=False, catalog_name='bollettino'): def run(use_saved=False, catalog_name='bollettino'):
years = [5, 10] years = [10]
test = 'T' test = 'T'
types = [True, False] ## Ordered IG matrix types = [True] ## Ordered IG matrix
models = {5: get_models.five_years(), models = {5: get_models.five_years(),
...@@ -152,10 +159,10 @@ def run(use_saved=False, catalog_name='bollettino'): ...@@ -152,10 +159,10 @@ def run(use_saved=False, catalog_name='bollettino'):
tw_figs_path = os.path.join(paths.csep_figs, 'tw_single') tw_figs_path = os.path.join(paths.csep_figs, 'tw_single')
os.makedirs(tw_figs_path, exist_ok=True) os.makedirs(tw_figs_path, exist_ok=True)
for yr in years: # for yr in years:
for model_i in range(len(models[yr])): # for model_i in range(len(models[yr])):
plot_single_results(Results[yr], years=yr, ref_model=model_i, # plot_single_results(Results[yr], years=yr, ref_model=model_i,
folder=tw_figs_path) # folder=tw_figs_path)
for yr, typ in itertools.product(years, types): for yr, typ in itertools.product(years, types):
plot_all_results(Results[yr], years=yr, order=typ, plot_all_results(Results[yr], years=yr, order=typ,
savepath=paths.get_csep_figpath(test, yr, type=catalog_name + typ * 'order')) savepath=paths.get_csep_figpath(test, yr, type=catalog_name + typ * 'order'))
...@@ -165,6 +172,6 @@ if __name__ =='__main__': ...@@ -165,6 +172,6 @@ if __name__ =='__main__':
use_saved = True use_saved = True
catalogs = ['bollettino', 'emrcmt', 'horus'] catalogs = ['bollettino', 'emrcmt', 'horus']
for cat in catalogs: for cat in catalogs[:1]:
run(use_saved, cat) run(use_saved, cat)
...@@ -6,16 +6,10 @@ import matplotlib.pyplot as plt ...@@ -6,16 +6,10 @@ import matplotlib.pyplot as plt
import numpy as np import numpy as np
import matplotlib.patches as patches import matplotlib.patches as patches
from matplotlib.lines import Line2D from matplotlib.lines import Line2D
import seaborn
seaborn.set_style("white", {"axes.facecolor": ".9", 'font.family': 'Ubuntu'})
def sig_exp(num_str):
parts = num_str.split('.', 2)
decimal = parts[1] if len(parts) > 1 else ''
exp = -len(decimal)
digits = parts[0].lstrip('0') + decimal
trimmed = digits.rstrip('0')
exp += len(digits) - len(trimmed)
sig = int(trimmed) if trimmed else 0
return sig, exp
def norm(array): def norm(array):
return (array - array.min())/(array.max() -array.min()) return (array - array.min())/(array.max() -array.min())
...@@ -44,22 +38,24 @@ def plot_axis(axis, n_results, offset, end_theta, n, min_y, ...@@ -44,22 +38,24 @@ def plot_axis(axis, n_results, offset, end_theta, n, min_y,
axis_angle = (end_theta + np.deg2rad(offset) + n*(2*np.pi-end_theta - 2*np.deg2rad(offset))/(n_results-1)) axis_angle = (end_theta + np.deg2rad(offset) + n*(2*np.pi-end_theta - 2*np.deg2rad(offset))/(n_results-1))
axis.plot([axis_angle, axis_angle], [min_y, 1. + min_y], axis.plot([axis_angle, axis_angle], [min_y, 1. + min_y],
color=color, linewidth=2, linestyle=':', alpha=0.5) color=color, linewidth=2, linestyle=':', alpha=0.5)
for i in yticks: for i in yticks:
val = (i - min_y) * (array.max() - array.min()) + array.min() val = (i - min_y) * (array.max() - array.min()) + array.min()
if 'e' in format: if 'e' in format:
# val = format % val val = format % val
print(val) # val, exp = sig_exp(str(val))
val, exp = sig_exp(str(val)) val, exp = str(val).split('e')
val = str(val)
else: else:
val = format % val val = format % val
if low_bound and i == yticks[0]: if low_bound and i == yticks[0]:
val = '<' + val val = '<' + val
axis.text(axis_angle, i - 0.03, val, axis.text(axis_angle, i - 0.03, val,
rotation=np.rad2deg(axis_angle) + 90, color=color, rotation=np.rad2deg(axis_angle) + 90, color=color,
ha='center', va='center', fontsize=fontsize) ha='center', va='center', fontsize=fontsize)
axis.text(axis_angle, i + 0.05, '%s' % label,
color=color, ha='left', va='center', fontsize=10)
# axis.text(axis_angle, i + 0.15, label, color=color, # axis.text(axis_angle, i + 0.15, label, color=color,
# ha='center', va='center', fontsize=14) # ha='center', va='center', fontsize=14)
...@@ -96,6 +92,7 @@ def plot_scores(arrays, colors, result_labels, model_labels, ...@@ -96,6 +92,7 @@ def plot_scores(arrays, colors, result_labels, model_labels,
theta = np.linspace(0.0, end_theta, n_results * n_models, endpoint=False) theta = np.linspace(0.0, end_theta, n_results * n_models, endpoint=False)
# Rearange results and colors # Rearange results and colors
tuple_arrays = tuple([norm(i) for i in arrays]) tuple_arrays = tuple([norm(i) for i in arrays])
score_total = np.vstack(tuple_arrays).T.ravel() score_total = np.vstack(tuple_arrays).T.ravel()
...@@ -111,6 +108,7 @@ def plot_scores(arrays, colors, result_labels, model_labels, ...@@ -111,6 +108,7 @@ def plot_scores(arrays, colors, result_labels, model_labels,
# Plot Data # Plot Data
ax.bar(theta + width/2., score_total, width=width, bottom=min_y, ax.bar(theta + width/2., score_total, width=width, bottom=min_y,
color=colors_full, alpha=1, zorder=0) color=colors_full, alpha=1, zorder=0)
...@@ -123,26 +121,26 @@ def plot_scores(arrays, colors, result_labels, model_labels, ...@@ -123,26 +121,26 @@ def plot_scores(arrays, colors, result_labels, model_labels,
yticks = plot_rticks(ax, min_y, ny) yticks = plot_rticks(ax, min_y, ny)
# todo make generic # todo make generic
print(angle_offset) new_labels = ['Log-likelihood', None, 'Brier score \n ($\cdot\,10^{-4}$)', None]
for k, i in enumerate([0, 2]): for k, i in enumerate([0, 2]):
plot_axis(ax, 2, offset, end_theta, k, min_y, arrays[i], yticks, lowbounds[i], plot_axis(ax, 2, offset, end_theta, k, min_y, arrays[i], yticks, lowbounds[i],
color_array[i], result_labels[i], fontsize, format[i]) color_array[i], new_labels[i], fontsize, format[i])
return ax return ax
def plot_legends(Axes, colors, labels): def plot_legends(Axes, colors, labels):
legend_elements = [Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[0]), legend_elements = [Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[0]),
lw=5, label=labels[0]), lw=4, label=labels[0]),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[1]), Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[1]),
lw=5, label=labels[1]), lw=4, label=labels[1]),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[2]), Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[2]),
lw=5, label=labels[2]), lw=4, label=labels[2]),
Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[3]), Line2D([0], [0], color=plt.cm.colors.to_rgb(colors[3]),
lw=5, label=labels[3]), lw=4, label=labels[3]),
Line2D([0], [0], color=colors[4], Line2D([0], [0], color=colors[4],
lw=0, marker='o', markersize=15, label=labels[4])] lw=0, marker='o', markersize=10, label=labels[4])]
Axes.get_figure().legend(handles=legend_elements, loc='lower right') Axes.get_figure().legend(handles=legend_elements, loc='lower right', fontsize=9)
return Axes return Axes
def plot_results(years, labels, p=0.01, lowcuts=False, show=True, def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
...@@ -160,7 +158,6 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True, ...@@ -160,7 +158,6 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
ll_label = r'$\mathcal{L}$' ll_label = r'$\mathcal{L}$'
if isinstance(lowcuts[0], (float, int)): if isinstance(lowcuts[0], (float, int)):
print('a')
ll_score[ll_score < lowcuts[0]] = lowcuts[0] ll_score[ll_score < lowcuts[0]] = lowcuts[0]
...@@ -168,14 +165,13 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True, ...@@ -168,14 +165,13 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
CL = paths.get_csep_result('CL', years, catalog) CL = paths.get_csep_result('CL', years, catalog)
cl_score = np.array([i.observed_statistic for i in CL]) cl_score = np.array([i.observed_statistic for i in CL])
cl_label = r'$\mathcal{L}{\|N}$' cl_label = r'$\mathcal{L}_{\|N}$'
if isinstance(lowcuts[1], (float, int)): if isinstance(lowcuts[1], (float, int)):
cl_score[cl_score < lowcuts[1]] = lowcuts[1] cl_score[cl_score < lowcuts[1]] = lowcuts[1]
## Brier score ## Brier score
Brier_results = paths.get_csep_result('B', years, catalog) Brier_results = paths.get_csep_result('B', years, catalog)
Brier = np.array([i.test_distribution[3] for i in Brier_results]) Brier = np.array([i.test_distribution[3] for i in Brier_results])
print(Brier)
brier_label = r'$\mathcal{B}$' brier_label = r'$\mathcal{B}$'
if isinstance(lowcuts[2], (float, int)): if isinstance(lowcuts[2], (float, int)):
Brier[Brier < lowcuts[2]] = lowcuts[2] Brier[Brier < lowcuts[2]] = lowcuts[2]
...@@ -183,7 +179,6 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True, ...@@ -183,7 +179,6 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
Brier_bin = np.array([i.test_distribution[2] for i in Brier_results]) Brier_bin = np.array([i.test_distribution[2] for i in Brier_results])
if isinstance(lowcuts[3], (float, int)): if isinstance(lowcuts[3], (float, int)):
Brier_bin[Brier_bin < lowcuts[3]] = lowcuts[3] Brier_bin[Brier_bin < lowcuts[3]] = lowcuts[3]
## Consistency tests ## Consistency tests
Consistency_Results = [] Consistency_Results = []
for n, m, s in zip(paths.get_csep_result('N', years, catalog), for n, m, s in zip(paths.get_csep_result('N', years, catalog),
...@@ -198,13 +193,11 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True, ...@@ -198,13 +193,11 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
model_cons.append('S') model_cons.append('S')
Consistency_Results.append(model_cons) Consistency_Results.append(model_cons)
ref_name = 'All models'
names = [i.sim_name for i in LL] names = [i.sim_name for i in LL]
## Order results in terms of ~average performance for visualization purpose ## Order results in terms of ~average performance for visualization purpose
order_val = norm(ll_score) + norm(cl_score) + norm(Brier_bin) + norm(Brier) order_val = norm(ll_score) + norm(cl_score) + norm(Brier_bin) + norm(Brier)
order = np.flip(np.argsort(order_val)) order = np.flip(np.argsort(order_val))
ll_score = ll_score[order] ll_score = ll_score[order]
...@@ -215,17 +208,21 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True, ...@@ -215,17 +208,21 @@ def plot_results(years, labels, p=0.01, lowcuts=False, show=True,
Consistency_Results = [Consistency_Results[i] for i in order] Consistency_Results = [Consistency_Results[i] for i in order]
## Complete array ## Complete array
colors = ['darkorange', 'orange', 'steelblue', 'blue', 'green'] colors = ['darkorange',
'darkred',
'teal',
'lightskyblue',
'olivedrab']
Axes = plot_scores([ll_score, cl_score, Brier, Brier_bin], Axes = plot_scores([ll_score, cl_score, Brier, Brier_bin],
colors[:4], colors[:4],
[ll_label, cl_label, brier_label, brier_bin_label], [ll_label, cl_label, brier_label, brier_bin_label],
model_names,format=format, angle_offset=60, offset=10, min_y=0.3, ny=4, model_names, format=format, angle_offset=60, offset=10, min_y=0.3, ny=4, fontsize=9,
lowbounds = [bool(i) for i in lowcuts]) lowbounds = [bool(i) for i in lowcuts])
Axes = plot_all_consistencies(Axes, Consistency_Results, color=colors[4]) Axes = plot_all_consistencies(Axes, Consistency_Results, color=colors[4])
Axes = plot_legends(Axes, colors, labels) Axes = plot_legends(Axes, colors, labels)
Axes.set_title('Test results - %i years (%s)' % (years, ref_name) , pad=15, fontsize=14) # Axes.set_title('Test results - %i years' % (years) , pad=15, fontsize=14)
if savepath: if savepath:
plt.savefig(savepath, dpi=300, transparent=True) plt.savefig(savepath, dpi=300, transparent=True)
plt.show() plt.show()
...@@ -262,8 +259,8 @@ if __name__ == '__main__': ...@@ -262,8 +259,8 @@ if __name__ == '__main__':
# savepath=paths.get_csep_figpath('Total', 10, 'ref11_CL_%s' % cat), catalog=cat) # savepath=paths.get_csep_figpath('Total', 10, 'ref11_CL_%s' % cat), catalog=cat)
plot_results(10, plot_results(10,
labels=labels, labels=labels,
format=['%i', '%i', '%.3e', '%.3e'], format=['%i', '%i', '%.4e', '%.4e'],
lowcuts=[-195, -195, -0.0001086, -0.0001086], lowcuts=[-200, -200, -0.00010855, -0.00010855],
savepath=paths.get_csep_figpath('Scores', 10, 'Total'), savepath=paths.get_csep_figpath('Scores', 10, 'Total_%s' % cat, 'svg'),
catalog=cat) catalog=cat)
...@@ -135,7 +135,7 @@ def get_kripley_result_path(method, year): ...@@ -135,7 +135,7 @@ def get_kripley_result_path(method, year):
:return: :return:
""" """
return join(kripley_store[int(year)], '%s_%s.txt' % (method, str(year))) return join(kripley_store[int(year)], '%s_%s' % (method, str(year)))
def get_csep_result(test, year, type=False): def get_csep_result(test, year, type=False):
...@@ -153,7 +153,7 @@ def get_csep_result(test, year, type=False): ...@@ -153,7 +153,7 @@ def get_csep_result(test, year, type=False):
def get_csep_figpath(test, year, type=None): def get_csep_figpath(test, year, type=None, format='png'):
""" """
Get the figure path of a particular test Get the figure path of a particular test
:param test: Name of the test :param test: Name of the test
...@@ -165,7 +165,7 @@ def get_csep_figpath(test, year, type=None): ...@@ -165,7 +165,7 @@ def get_csep_figpath(test, year, type=None):
type = '_' + str(type) type = '_' + str(type)
else: else:
type = '' type = ''
return join(csep_figs, "%s_%s%s.png" % (test, str(year), type)) return join(csep_figs, "%s_%s%s.%s" % (test, str(year), type, format))
def get_kripley_figpath(analysis, year, model): def get_kripley_figpath(analysis, year, model):
""" """
......
results/figures/csep/T_5_emrcmt.png

248 KB | W: | H:

results/figures/csep/T_5_emrcmt.png

270 KB | W: | H:

results/figures/csep/T_5_emrcmt.png
results/figures/csep/T_5_emrcmt.png
results/figures/csep/T_5_emrcmt.png
results/figures/csep/T_5_emrcmt.png
  • 2-up
  • Swipe
  • Onion skin
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment